<repo_name>xuewengeophysics/xwStudyNLP<file_sep>/text_classification/Transformer/model/Transformer.py
import tensorflow as tf
import math
import numpy as np
from tensorflow.python.ops import array_ops


class Muti_head_Attention(object):

    def __init__(self, max_length, num_classes, vocab_size, embedding_size, hidden_num, num_blocks, num_heads):
        # placeholders for input output and dropout
        self.input_x = tf.placeholder(tf.int32, shape=[None, max_length], name='input_x')
        self.input_y = tf.placeholder(tf.int32, shape=[None, num_classes], name='input_y')
        self.drop_out_prob = tf.placeholder(tf.float32, name='drop_out_keep')

        # embedding layer
        with tf.device('/gpu:0'), tf.name_scope("embedding"):
            self.embedding = tf.Variable(tf.truncated_normal([vocab_size, embedding_size],
                                         stddev=1.0 / math.sqrt(embedding_size)),
                                         name="embedding", trainable=True)
            self.embedding_chars = tf.nn.embedding_lookup(self.embedding, self.input_x)

        # Positional Encoding
        N = array_ops.shape(self.embedding_chars)[0]
        T = max_length
        self.embedding_chars += self.positional_encoding(N, T,
                                                         num_units=hidden_num,
                                                         zero_pad=False,
                                                         scale=False,
                                                         scope="enc_pe")

        # Dropout
        self.enc = tf.layers.dropout(self.embedding_chars, rate=self.drop_out_prob)

        # Blocks
        for i in range(num_blocks):
            with tf.variable_scope("num_blocks_{}".format(i)):
                # Multihead Attention
                self.enc = self.multihead_attention(queries=self.enc,
                                                    keys=self.enc,
                                                    num_units=hidden_num,
                                                    num_heads=num_heads,
                                                    dropout_rate=self.drop_out_prob,
                                                    causality=False)

                # Feed Forward
                self.enc = self.feedforward(self.enc, num_units=[4 * hidden_num, hidden_num])

        # 将特征进行拼接
        self.enc = tf.reshape(self.enc, [-1, max_length * hidden_num, 1])
        self.enc = tf.squeeze(self.enc, -1)

        fc_w = tf.Variable(tf.truncated_normal([max_length * hidden_num, num_classes], stddev=0.1), name='fc_w')
        fc_b = tf.Variable(tf.zeros([num_classes]), name='fc_b')
        # 定义损失函数
        l2_loss = 0
        l2_loss += tf.nn.l2_loss(fc_w)
        l2_loss += tf.nn.l2_loss(fc_b)

        self.logits = tf.matmul(self.enc, fc_w) + fc_b
        self.score = tf.nn.softmax(self.logits, name='score')
        self.predictions = tf.argmax(self.score, 1, name="predictions")

        self.cost = tf.losses.softmax_cross_entropy(self.input_y, self.logits)
        # l2_reg_lambda = 0.01
        # self.cost = self.cost + l2_reg_lambda * l2_loss

        tvars = tf.trainable_variables()
        grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tvars), 5)

        optimizer = tf.train.AdamOptimizer(0.001)
        self.train_op = optimizer.apply_gradients(zip(grads, tvars))

        self.accuracy = tf.reduce_mean(
                        tf.cast(tf.equal(tf.argmax(self.input_y, axis=1), tf.argmax(self.score, axis=1)), tf.float32))

    def positional_encoding(self, N, T,
                            num_units,
                            zero_pad=True,
                            scale=True,
                            scope="positional_encoding",
                            reuse=None):
        '''Sinusoidal Positional_Encoding.

        Args:
          inputs: A 2d Tensor with shape of (N, T).
          num_units: Output dimensionality
          zero_pad: Boolean. If True, all the values of the first row (id = 0) should be constant zero
          scale: Boolean. If True, the output will be multiplied by sqrt num_units(check details from paper)
          scope: Optional scope for `variable_scope`.
          reuse: Boolean, whether to reuse the weights of a previous layer
            by the same name.

        Returns:
            A 'Tensor' with one more rank than inputs's, with the dimensionality should be 'num_units'
        '''
        with tf.variable_scope(scope, reuse=reuse):
            position_ind = tf.tile(tf.expand_dims(tf.range(T), 0), [N, 1])  # tf.expand_dims(tf.range(T), 0) --> [1, T]
                                                                            # tf.tile([1, T], [N, 1]) --> [N, T]
                                                                            # N为batch_size，T为最长句子长度

            # First part of the PE function: sin and cos argument
            position_enc = np.array([
                [pos / np.power(10000, 2. * i / num_units) for i in range(num_units)]
                for pos in range(T)])  # dim [T, num_units], num_units=hidden_num

            # Second part, apply the cosine to even columns and sin to odds.
            position_enc[:, 0::2] = np.sin(position_enc[:, 0::2])  # dim 2i
            position_enc[:, 1::2] = np.cos(position_enc[:, 1::2])  # dim 2i+1

            # Convert to a tensor
            lookup_table = tf.convert_to_tensor(position_enc)

            if zero_pad:
                lookup_table = tf.concat((tf.zeros(shape=[1, num_units]),
                                          lookup_table[1:, :]), 0)
            outputs = tf.nn.embedding_lookup(lookup_table, position_ind)

            if scale:
                outputs = outputs * num_units ** 0.5

            outputs = tf.cast(outputs, dtype=tf.float32)

            return outputs

    def multihead_attention(self, queries,
                            keys,
                            num_units=None,
                            num_heads=8,
                            dropout_rate=0,
                            causality=False,
                            scope="multihead_attention",
                            reuse=None):
        '''Applies multihead attention.

        Args:
          queries: A 3d tensor with shape of [N, T_q, C_q].
          keys: A 3d tensor with shape of [N, T_k, C_k].
          num_units: A scalar. Attention size.
          dropout_rate: A floating point number.
          is_training: Boolean. Controller of mechanism for dropout.
          causality: Boolean. If true, units that reference the future are masked.
          num_heads: An int. Number of heads.
          scope: Optional scope for `variable_scope`.
          reuse: Boolean, whether to reuse the weights of a previous layer
            by the same name.

        Returns
          A 3d tensor with shape of (N, T_q, C)
        '''
        with tf.variable_scope(scope, reuse=reuse):
            # Set the fall back option for num_units
            if num_units is None:
                num_units = queries.get_shape().as_list[-1]

            # Linear projections
            Q = tf.layers.dense(queries, num_units, activation=tf.nn.relu)  # (N, T_q, C)
            K = tf.layers.dense(keys, num_units, activation=tf.nn.relu)  # (N, T_k, C)
            V = tf.layers.dense(keys, num_units, activation=tf.nn.relu)  # (N, T_k, C)

            # Split and concat
            Q_ = tf.concat(tf.split(Q, num_heads, axis=2), axis=0)  # (h*N, T_q, C/h)
            K_ = tf.concat(tf.split(K, num_heads, axis=2), axis=0)  # (h*N, T_k, C/h)
            V_ = tf.concat(tf.split(V, num_heads, axis=2), axis=0)  # (h*N, T_k, C/h)

            # Multiplication
            outputs = tf.matmul(Q_, tf.transpose(K_, [0, 2, 1]))  # (h*N, T_q, T_k)

            # Scale
            outputs = outputs / (K_.get_shape().as_list()[-1] ** 0.5)

            # Key Masking
            key_masks = tf.sign(tf.abs(tf.reduce_sum(keys, axis=-1)))  # (N, T_k)
            key_masks = tf.tile(key_masks, [num_heads, 1])  # (h*N, T_k)
            key_masks = tf.tile(tf.expand_dims(key_masks, 1), [1, tf.shape(queries)[1], 1])  # (h*N, T_q, T_k)

            paddings = tf.ones_like(outputs) * (-2 ** 32 + 1)
            outputs = tf.where(tf.equal(key_masks, 0), paddings, outputs)  # (h*N, T_q, T_k)

            # Causality = Future blinding
            if causality:
                diag_vals = tf.ones_like(outputs[0, :, :])  # (T_q, T_k)
                tril = tf.linalg.LinearOperatorLowerTriangular(diag_vals).to_dense()  # (T_q, T_k)
                masks = tf.tile(tf.expand_dims(tril, 0), [tf.shape(outputs)[0], 1, 1])  # (h*N, T_q, T_k)

                paddings = tf.ones_like(masks) * (-2 ** 32 + 1)
                outputs = tf.where(tf.equal(masks, 0), paddings, outputs)  # (h*N, T_q, T_k)

            # Activation
            outputs = tf.nn.softmax(outputs)  # (h*N, T_q, T_k)

            # Query Masking
            query_masks = tf.sign(tf.abs(tf.reduce_sum(queries, axis=-1)))  # (N, T_q)
            query_masks = tf.tile(query_masks, [num_heads, 1])  # (h*N, T_q)
            query_masks = tf.tile(tf.expand_dims(query_masks, -1), [1, 1, tf.shape(keys)[1]])  # (h*N, T_q, T_k)
            outputs *= query_masks  # broadcasting. (N, T_q, C)

            # Dropouts
            outputs = tf.layers.dropout(outputs, rate=dropout_rate)

            # Weighted sum
            outputs = tf.matmul(outputs, V_)  # ( h*N, T_q, C/h)

            # Restore shape
            outputs = tf.concat(tf.split(outputs, num_heads, axis=0), axis=2)  # (N, T_q, C)

            # Residual connection
            outputs += queries

            # Normalize
            outputs = self.normalize(outputs)  # (N, T_q, C)

        return outputs

    def feedforward(self, inputs,
                    num_units=[2048, 512],
                    scope="multihead_attention",
                    reuse=None):
        '''Point-wise feed forward net.

        Args:
          inputs: A 3d tensor with shape of [N, T, C].
          num_units: A list of two integers.
          scope: Optional scope for `variable_scope`.
          reuse: Boolean, whether to reuse the weights of a previous layer
            by the same name.

        Returns:
          A 3d tensor with the same shape and dtype as inputs
        '''
        with tf.variable_scope(scope, reuse=reuse):
            # Inner layer
            params = {"inputs": inputs, "filters": num_units[0], "kernel_size": 1,
                      "activation": tf.nn.relu, "use_bias": True}
            outputs = tf.layers.conv1d(**params)

            # Readout layer
            params = {"inputs": outputs, "filters": num_units[1], "kernel_size": 1,
                      "activation": None, "use_bias": True}
            outputs = tf.layers.conv1d(**params)

            # Residual connection
            outputs += inputs

            # Normalize
            outputs = self.normalize(outputs)

        return outputs

    def normalize(self, inputs,
                  epsilon=1e-8,
                  scope="ln",
                  reuse=None):
        '''Applies layer normalization.

        Args:
          inputs: A tensor with 2 or more dimensions, where the first dimension has
            `batch_size`.
          epsilon: A floating number. A very small number for preventing ZeroDivision Error.
          scope: Optional scope for `variable_scope`.
          reuse: Boolean, whether to reuse the weights of a previous layer
            by the same name.

        Returns:
          A tensor with the same shape and data dtype as `inputs`.
        '''
        with tf.variable_scope(scope, reuse=reuse):
            inputs_shape = inputs.get_shape()
            params_shape = inputs_shape[-1:]

            mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)
            beta = tf.Variable(tf.zeros(params_shape))
            gamma = tf.Variable(tf.ones(params_shape))
            normalized = (inputs - mean) / ((variance + epsilon) ** (.5))
            outputs = gamma * normalized + beta

        return outputs
<file_sep>/XLNet/README.md
# XLNet: Generalized Autoregressive Pretraining for Language Understanding

参考资料
==
1  [论文BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805)<br>
2  [代码BERT: zihangdai/xlnet](https://github.com/zihangdai/xlnet)<br>
<file_sep>/text_classification/HierarchicalAttentionNetwork/HanModel.py
# -*- coding: utf-8 -*-
"""
HierarchicalAttention:
1.Word Encoder
2.Word Attention
3.Sentence Encoder
4.Sentence Attention
5.linear classifier
"""

import tensorflow as tf
import numpy as np
import math

class HierarchicalAttention(object):
    def __init__(self, config):
        self.config = config

        # placeholders for input output and dropout
        # input_x的维度是[batch_size, sequence_length]
        # num_sentences在原文中用 L 表示，num_words在原文中用 T 表示，num_words * num_sentences = sequence_length
        # input_x的维度是[batch_size, num_classes]
        self.input_x = tf.placeholder(tf.int32, shape=[None, self.config.sequence_length], name='input_x')
        self.input_y = tf.placeholder(tf.int32, shape=[None, self.config.num_classes], name='input_y')
        self.dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob')
        self.global_step = tf.Variable(0, trainable=False, name='global_step')

        # word embedding
        with tf.name_scope("embedding"):
            # 将输入的序列进行拆分，拆分成num_sentences个句子
            # 得到了num_sentences个维度为[None, num_words]的张量
            input_x = tf.split(self.input_x, self.config.num_sentences, axis=1)  # 在某个维度上split
            # 矩阵拼接，得到的张量维度为[None, self.num_sentences, num_words]
            input_x = tf.stack(input_x, axis=1)
            self.embedding = tf.get_variable("embeddings", shape=[self.config.vocab_size, self.config.embedding_size],
                                             initializer=tf.constant_initializer(self.config.pre_training),
                                             trainable=True) # 是否使用预训练词向量，静态(False)或动态(默认True)
            # 张量维度为[None, num_sentences, num_words, embedding_size]
            embedding_vectors = tf.nn.embedding_lookup(self.embedding, input_x)
            # 输入到word encoder层的张量的维度为[batch_size*num_sentences, num_words, embedding_size]
            self.embedding_inputs = tf.reshape(embedding_vectors, shape=[-1, self.config.num_words, self.config.embedding_size])

        # word encoder
        with tf.name_scope("word_encoder"):
            # 给定文本词向量x(it)，得到正向隐藏状态h(it)、反向隐藏状态h(it)
            # 输出的隐藏状态张量的维度为[batch_size*num_sentences, num_words, hidden_size]
            hidden_state_fw, hidden_state_bw = self.build_bidirectional_rnn(self.embedding_inputs, "word_encoder")
            # 拼接得到h(it) = [正向h(it), 反向h(it)]，维度为[batch_size*num_sentences, num_words, hidden_size*2]
            word_hidden_state = tf.concat((hidden_state_fw, hidden_state_bw), 2)

        # word attention
        with tf.name_scope("word_attention"):
            # 得到sentence_vector s(i)=sum(alpha(it)*h(it))
            # 张量维度为[batch_size*num_sentences, hidden_size*2]
            sentence_vector = self.build_attention(word_hidden_state, "word_attention")

        # sentence encoder
        with tf.name_scope("sentence_encoder"):
            # 句子级输入的是句子，reshape得到维度为[batch_size, num_sentences, hidden_size*2]的张量
            sentence_vector = tf.reshape(sentence_vector, shape=[-1, self.config.num_sentences, self.config.hidden_size * 2])
            # 给定句子级向量s(i)，得到正向隐藏状态h(i)、反向隐藏状态h(i)
            # 输出的隐藏状态张量的维度为[batch_size, num_sentences, hidden_size]
            hidden_state_fw, hidden_state_bw = self.build_bidirectional_rnn(sentence_vector, "sentence_encoder")
            # 拼接得到h(i) = [正向h(i), 反向h(i)]，维度为[batch_size, num_sentences, hidden_size*2]
            sentence_hidden_state = tf.concat((hidden_state_fw, hidden_state_bw), 2)

        # sentence attention
        with tf.name_scope("sentence_attention"):
            # 得到document_vector v=sum(alpha(i)*h(i))
            # 张量维度为[batch_size, hidden_size * 2]
            document_vector = self.build_attention(sentence_hidden_state, "sentence_attention")

        # dropout
        with tf.name_scope("dropout"):
            h_drop = tf.nn.dropout(document_vector, self.dropout_keep_prob)

        # classifier
        with tf.name_scope("output"):
            # 添加一个全连接层
            self.logits = tf.layers.dense(h_drop, self.config.num_classes, name='fc2')
            # 预测类别
            self.predict = tf.argmax(tf.nn.softmax(self.logits), 1, name="prediction")

        # loss
        with tf.name_scope('loss'):
            losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, labels=self.input_y)
            self.loss = tf.reduce_mean(losses)

        # optimizer
        with tf.name_scope('optimizer'):
            optimizer = tf.train.AdamOptimizer(self.config.learning_rate)
            gradients, variables = zip(*optimizer.compute_gradients(self.loss))  # 计算变量梯度，得到梯度值、变量
            gradients, _ = tf.clip_by_global_norm(gradients, self.config.gradient_clip)
            self.optimizer = optimizer.apply_gradients(zip(gradients, variables), global_step=self.global_step)
            # global_step 自动+1

        # accuracy
        with tf.name_scope('accuracy'):
            correct_prediction = tf.equal(self.predict, tf.argmax(self.input_y, 1))
            self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')


    def rnn_cell(self):
        """获取rnn的cell，可选RNN、LSTM、GRU"""
        if self.config.rnn_type == "Vanilla":
            return tf.nn.rnn_cell.BasicRNNCell(self.config.hidden_size)
        elif self.config.rnn_type == "LSTM":
            return tf.nn.rnn_cell.BasicLSTMCell(self.config.hidden_size)
        elif self.config.rnn_type == "GRU":
            return tf.nn.rnn_cell.GRUCell(self.config.hidden_size)
        else:
            raise Exception("rnn_type must be Vanilla、LSTM or GRU!")

    def build_bidirectional_rnn(self, inputs, name):
        with tf.variable_scope(name):
            fw_cell = self.rnn_cell()
            fw_cell = tf.nn.rnn_cell.DropoutWrapper(fw_cell, output_keep_prob=self.config.dropout_keep_prob)
            bw_cell = self.rnn_cell()
            bw_cell = tf.nn.rnn_cell.DropoutWrapper(bw_cell, output_keep_prob=self.config.dropout_keep_prob)
            (output_fw, output_bw), states = tf.nn.bidirectional_dynamic_rnn(cell_fw=fw_cell,
                                                                             cell_bw=bw_cell,
                                                                             inputs=inputs,
                                                                             dtype=tf.float32)
        return output_fw, output_bw

    def build_attention(self, inputs, name):
        with tf.variable_scope(name):
            # inputs词级h(it)的维度为[batch_size*num_sentences, num_words, hidden_size*2]
            # inputs句子级h(i)的维度为[batch_size, num_sentences, hidden_size*2]
            # 采用general形式计算权重，采用单层神经网络来给出attention中的score得分
            hidden_vec = tf.layers.dense(inputs, self.config.hidden_size * 2, activation=tf.nn.tanh, name='u_hidden')
            u_key = tf.Variable(tf.truncated_normal([self.config.hidden_size * 2]), name='u_key')
            # alpha就是attention中的score得分
            alpha = tf.nn.softmax(tf.reduce_sum(tf.multiply(hidden_vec, u_key), axis=2, keep_dims=True), dim=1)
            # 对隐藏状态进行加权
            attention_output = tf.reduce_sum(tf.multiply(inputs, alpha), axis=1)
        return attention_output
<file_sep>/text_classification/TextCNN/model/Textcnn.py
#encoding:utf-8
import tensorflow as tf

class Textcnn(object):

    def __init__(self, config):
        config = config
        # placeholders for input output and dropout
        self.input_x = tf.placeholder(tf.int32, shape=[None, config.seq_length], name='input_x')
        self.input_y = tf.placeholder(tf.float32, shape=[None, config.num_classes], name='input_y')
        self.dropout_keep_prob = tf.placeholder(tf.float32, name='dropout')
        self.global_step = tf.Variable(0, trainable=False, name='global_step')
        # global_step 代表全局步数，比如在多少步该进行什么操作，现在神经网络训练到多少轮等等，类似于一个钟表。
        l2_loss = tf.constant(0.0)

        # embedding layer
        with tf.device('/gpu:0'):
            self.embedding = tf.get_variable("embeddings", shape=[config.vocab_size, config.embedding_size],
                                             initializer=tf.constant_initializer(config.pre_training),
                                             trainable=False) # 是否使用预训练词向量，静态(False)或动态(默认True)
            embedding_input = tf.nn.embedding_lookup(self.embedding, self.input_x)
            # conv2d操作接收一个4维的张量，维度分别代表batch, width, height 和 channel，这里的embedding不包含channel维度
            # 因此要手动添加，得到形状为[None, sequence_length, embedding_size, 1]的embedding层。
            self.embedding_expand = tf.expand_dims(embedding_input, -1)

        # 添加卷积层、池化层
        pooled_outputs = []
        for i, filter_size in enumerate(config.filter_sizes):
            with tf.name_scope("conv-maxpool-%s" % filter_size):
                # CNN layer
                # conv2d的卷积核形状[filter_height, filter_width, in_channels, out_channels]
                filter_shape = [filter_size, config.embedding_size, 1, config.num_filters]  # 卷积核维度

                # 设定卷积核的参数 W b
                W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name= 'W')  # 图变量初始化，截断正态分布
                b = tf.Variable(tf.constant(0.1, shape=[config.num_filters]), name='b')  # 生成常量
                conv = tf.nn.conv2d(self.embedding_expand, W, strides=[1, 1, 1, 1], padding='VALID', name='conv')
                # nonlinearity activate funtion
                h = tf.nn.relu(tf.nn.bias_add(conv, b), name='relu')

                # global max pooling layer
                # 每个filter的max-pool输出形状:[batch_size, 1, 1, num_filters]
                pooled = tf.nn.max_pool(h,
                                        ksize=[1, config.seq_length - filter_size + 1, 1, 1],
                                        strides=[1, 1, 1, 1],
                                        padding='VALID',
                                        name='pool')
                pooled_outputs.append(pooled)

        # combine all the pooled features
        # 有三种卷积核尺寸， 每个卷积核有num_filters个channel，因此一共有 num_filters * len(filter_sizes)个
        num_filter_total = config.num_filters * len(config.filter_sizes)
        self.h_pool = tf.concat(pooled_outputs, 3)  # 在第3个维度进行张量拼接
        self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filter_total])

        # add dropout
        with tf.name_scope('dropout'):
            self.h_drop = tf.nn.dropout(self.h_pool_flat, self.dropout_keep_prob)

        # final (unnormalized) scores and predictions
        with tf.name_scope('output'):
            W_fc = tf.get_variable("W_fc", shape=[num_filter_total, config.num_classes],
                                   initializer=tf.contrib.layers.xavier_initializer())
            b_fc = tf.Variable(tf.constant(0.1, shape=[config.num_classes]), name='b_fc')
            self.logits = tf.matmul(self.h_drop, W_fc) + b_fc
            self.pro = tf.nn.softmax(self.logits)
            self.predicitions = tf.argmax(self.pro, 1, name='predictions')

        # calculate mean cross-entropy loss
        with tf.name_scope('loss'):
            # 损失函数，交叉熵
            loss = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, labels=self.input_y)
            self.loss = tf.reduce_mean(loss)  # 对交叉熵取均值非常有必要
            l2_loss += tf.nn.l2_loss(W_fc)
            l2_loss += tf.nn.l2_loss(b_fc)
            self.loss = tf.reduce_mean(loss) + config.l2_reg_lambda * l2_loss

        # 优化器
        with tf.name_scope('optimizer'):
            optimizer = tf.train.AdamOptimizer(config.learning_rate)
            gradients, variables = zip(*optimizer.compute_gradients(self.loss))  # 计算变量梯度，得到梯度值,变量
            gradients, _ = tf.clip_by_global_norm(gradients, config.clip)
            #对g进行l2正则化计算，比较其与clip的值，如果l2后的值更大，让梯度*(clip/l2_g),得到新梯度
            self.optimizer = optimizer.apply_gradients(zip(gradients, variables), global_step=self.global_step)

        # accuracy
        with tf.name_scope('accuracy'):
            correct_predictions = tf.equal(tf.argmax(self.input_y, 1), self.predicitions)
            self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, 'float32'), name='accuracy')
<file_sep>/word2vec/gensim_word2vec.py
"""
主要参数介绍如下：
01) sentences：语料，可以是一个列表，或者从文件中遍历读出（word2vec.LineSentence(filename)）。
02) size：词向量的维度，默认值是100。这个维度的取值一般与我们的语料的大小相关；
    如果是不大的语料，比如小于100M的文本语料，则使用默认值一般就可以了；如果是超大的语料，建议增大维度。
03) window：即词向量上下文最大距离，window越大，则和某一词较远的词也会产生上下文关系。
    默认值为5，在实际使用中，可以根据实际的需求来动态调整这个window的大小。
　　如果是小语料则这个值可以设的更小。对于一般的语料这个值推荐在[5, 10]之间。
04) sg：如果是0则是CBOW模型；是1则是Skip-Gram模型；默认是0即CBOW模型。
05) hs：即word2vec的两个解法。如果是0，则是Negative Sampling；
    是1并且负采样个数negative大于0，则是Hierarchical Softmax。默认是0即Negative Sampling。
06) negative：即使用Negative Sampling时负采样的个数，默认是5。推荐在[3, 10]之间。
07) cbow_mean：仅用于CBOW在做投影的时候，为0，则算法中的xw为上下文的词向量之和，为1则为上下文的词向量的平均值。默认值也是1。
08) min_count：需要计算词向量的最小词频。这个值可以去掉一些很生僻的低频词，默认是5。如果是小语料，可以调低这个值。
09) iter：随机梯度下降法中迭代的最大次数，默认是5。对于大语料，可以增大这个值。
10) alpha：在随机梯度下降法中迭代的初始步长，默认是0.025。
11) min_alpha: 由于算法支持在迭代的过程中逐渐减小步长，min_alpha给出了最小的迭代步。
"""
gensim.models.word2vec.Word2Vec(
    sentences=None, 
    corpus_file=None, 
    size=100, 
    alpha=0.025, 
    window=5, 
    min_count=5, 
    max_vocab_size=None, 
    sample=0.001, 
    seed=1, 
    workers=3, 
    min_alpha=0.0001, 
    sg=0, 
    hs=0, 
    negative=5, 
    ns_exponent=0.75, 
    cbow_mean=1, 
    hashfxn=<built-in function hash>, 
    iter=5, 
    null_word=0, 
    trim_rule=None, 
    sorted_vocab=1, 
    batch_words=10000, 
    compute_loss=False, 
    callbacks=(), 
    max_final_vocab=None)
    
<file_sep>/named_entity_recognition/README.md
# Named Entity Recognition

## 参考资料

1  [源码Determined22/zh-NER-TF](https://github.com/Determined22/zh-NER-TF)  [对应博客BiLSTM-CRF模型做基于字的中文命名实体识别](https://www.cnblogs.com/Determined22/p/7238342.html)<br>
2  [源码jiesutd/LatticeLSTM](https://github.com/jiesutd/LatticeLSTM)<br>
3  [源码guillaumegenthial/tf_ner](https://github.com/guillaumegenthial/tf_ner)<br>
4  [博客Guillaume Genthial blog/Sequence Tagging with Tensorflow](https://guillaumegenthial.github.io/sequence-tagging-with-tensorflow.html)  [对应源码guillaumegenthial/sequence_tagging](https://github.com/guillaumegenthial/sequence_tagging)<br>
5  [源码liuhuanyong/MedicalNamedEntityRecognition](https://github.com/liuhuanyong/MedicalNamedEntityRecognition)<br>
6  [源码zyh961117/CCKS-2017-task2](https://github.com/zyh961117/CCKS-2017-task2)<br>
<file_sep>/text_classification/TextCNN/README.md
# TextCNN

本文是参考gaussic大牛的“text-classification-cnn-rnn”后，基于同样的数据集，嵌入词级别所做的CNN文本分类实验结果，gaussic大牛是基于字符级的；<br><br>
进行了第二版的更新：1.加入不同的卷积核；2.加入正则化；3.词仅为中文或英文，删掉文本中数字、符号等类型的词；4.删除长度为1的词；<br>
<br>
训练结果较第一版有所提升，验证集准确率从96.5%达到97.8%，测试准备率从96.7%达到97.2%。<br>
<br>


本实验的主要目是为了探究基于Word2vec训练的词向量嵌入CNN后，对模型的影响，实验结果得到的模型在验证集达到97.8%的效果，gaussic大牛为94.12%；<br><br>
更多详细可以阅读gaussic大牛的博客：[text-classification-cnn-rnn](https://github.com/gaussic/text-classification-cnn-rnn)<br><br>

使用TextCNN进行中文文本分类

## 数据预处理流程
![image](https://github.com/xuewengeophysics/xwStudyNLP/blob/master/text_classification/TextCNN/images/%E6%95%B0%E6%8D%AE%E9%A2%84%E5%A4%84%E7%90%86%E6%B5%81%E7%A8%8B.png)

## TextCNN模型网络结构
![image](https://github.com/xuewengeophysics/xwStudyNLP/blob/master/text_classification/TextCNN/images/TextCNN%E6%A8%A1%E5%9E%8B.png)

## 参考资料

1 [文献Convolutional Neural Networks for Sentence Classification](https://arxiv.org/abs/1408.5882)<br>
2 [文献Character-level Convolutional Networks for Text Classification](https://arxiv.org/abs/1509.01626)<br>
3 [大牛dennybritz的博客Implementing a CNN for Text Classification in TensorFlow](http://www.wildml.com/2015/12/implementing-a-cnn-for-text-classification-in-tensorflow/)<br>
4 [源码dennybritz/cnn-text-classification-tf](https://github.com/dennybritz/cnn-text-classification-tf)<br>
5 [源码gaussic/text-classification-cnn-rnn](https://github.com/gaussic/text-classification-cnn-rnn)<br>
6 [源码cjymz886/text-cnn](https://github.com/cjymz886/text-cnn)<br>
7 [源码NLPxiaoxu/Easy_TextCnn_Rnn](https://github.com/NLPxiaoxu/Easy_TextCnn_Rnn)<br>
8 [源码YCG09/tf-text-classification](https://github.com/YCG09/tf-text-classification)<br>
9 [源码pengming617/text_classification](https://github.com/pengming617/text_classification)<br>
<file_sep>/Transformer/README.md
# transformer

## 参考资料

论文1：[Attention Is All You Need](https://arxiv.org/pdf/1706.03762.pdf)<br>
论文2：[Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860)<br>
源码1：[kimiyoung/transformer-xl](https://github.com/kimiyoung/transformer-xl)<br>

博客1：[<NAME>/The Illustrated Transformer](https://jalammar.github.io/illustrated-transformer)<br>
博客2：[<NAME>/Dissecting BERT Part 1: The Encoder](https://medium.com/dissecting-bert/dissecting-bert-part-1-d3c3d495cdb3)<br>
博客3：[harvardnlp/The Annotated Transformer](http://nlp.seas.harvard.edu/2018/04/03/attention.html)<br>

博客4：[张俊林/放弃幻想，全面拥抱Transformer：自然语言处理三大特征抽取器（CNN/RNN/TF）比较](https://zhuanlan.zhihu.com/p/54743941)

博客5：[机器翻译模型Transformer代码详细解析](https://blog.csdn.net/mijiaoxiaosan/article/details/74909076)
博客5对应源码：[Kyubyong/transformer](https://github.com/Kyubyong/transformer)

博客6：[文本分类实战之Transformer模型](https://www.cnblogs.com/jiangxinyang/p/10210813.html)
博客6对应源码：[jiangxinyang227/textClassifier/Transformer](https://github.com/jiangxinyang227/textClassifier/tree/master/Transformer)<br>

源码1：https://github.com/brightmart/text_classification/tree/master/a07_Transformer<br>
源码2：https://github.com/pengming617/text_classification/tree/master/model/transformer<br>
源码3：https://github.com/SamLynnEvans/Transformer
对应博客[<NAME>/How to code The Transformer in Pytorch](https://towardsdatascience.com/how-to-code-the-transformer-in-pytorch-24db27c8f9ec)
中文翻译[大数据文摘/百闻不如一码！手把手教你用Python搭一个Transformer](https://mp.weixin.qq.com/s/zLptc5bvo3rY_Jvu-rA6Pg)<br>


<file_sep>/README.md
# Natural Language Processing

NLP任务总结
=
![image](https://github.com/xuewengeophysics/xwStudyNLP/blob/master/images/NLP%E4%BB%BB%E5%8A%A1%E6%80%BB%E7%BB%93.jpg)

NLP常用算法
=
![image](https://github.com/xuewengeophysics/xwStudyNLP/blob/master/images/NLP%E5%B8%B8%E7%94%A8%E7%AE%97%E6%B3%95.jpg)

## 参考资料

1 [大牛<NAME>的博客](https://jalammar.github.io/)<br>
2 [大牛<NAME>博客](http://ruder.io/)<br>

## 中文NLP
1 [中文自然语言处理相关资料crownpku/Awesome-Chinese-NLP](https://github.com/crownpku/Awesome-Chinese-NLP)<br>
2 [大规模中文自然语言处理语料brightmart/nlp_chinese_corpus](https://github.com/brightmart/nlp_chinese_corpus)<br>
<file_sep>/text_classification/TextRCNN/TextRCnnModel.py
"""
参考资料：
[1]  https://github.com/roomylee/rcnn-text-classification
"""
import tensorflow as tf

class TextRCnn(object):
    """
    Recurrent Convolutional neural network for text classification
    1. embeddding layer,
    2. Bi-LSTM layer,
    3. max pooling,
    4. FC layer
    5. softmax
    """
    def __init__(self, config):
        self.config = config

        # placeholders for input output and dropout
        # input_x的维度是[batch_size, sequence_length]
        # input_y的维度是[batch_size, num_classes]
        self.input_x = tf.placeholder(tf.int32, shape=[None, self.config.sequence_length], name='input_x')
        self.input_y = tf.placeholder(tf.int32, shape=[None, self.config.num_classes], name='input_y')
        self.dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob')
        self.global_step = tf.Variable(0, trainable=False, name='global_step')

        l2_loss = tf.constant(0.0)
        text_length = self._length(self.input_x)

        # 1.get emebedding of words in the sentence
        with tf.name_scope("embedding"):
            # self.embedding = tf.Variable(tf.random_uniform([self.config.vocab_size, self.config.word_embedding_size], -1.0, 1.0),
            #                              name="embedding", trainable=True)
            # self.embedding = tf.Variable(tf.truncated_normal([self.config.vocab_size, self.config.word_embedding_size],
            #                              stddev=1.0 / math.sqrt(self.config.word_embedding_size)),
            #                              name="embedding", trainable=True)
            self.embedding = tf.get_variable("embeddings", shape=[self.config.vocab_size, self.config.word_embedding_size],
                                             initializer=tf.constant_initializer(self.config.pre_training),
                                             trainable=True) # 是否使用预训练词向量，静态(False)或动态(默认True)
            # 张量维度为[None, sequence_length, word_embedding_size]
            self.embedding_words = tf.nn.embedding_lookup(self.embedding, self.input_x)

        # 2. Bidirectional(Left&Right) Recurrent Structure
        # 输出张量 x_i = [c_l(w_i); e(w_i); c_r(w_i)], shape:[None, sequence_length, embedding_size*3]
        with tf.name_scope("bi-rnn"):
            fw_cell = self._get_cell(self.config.context_embedding_size, self.config.rnn_type)
            fw_cell = tf.nn.rnn_cell.DropoutWrapper(fw_cell, output_keep_prob=self.dropout_keep_prob)
            bw_cell = self._get_cell(self.config.context_embedding_size, self.config.rnn_type)
            bw_cell = tf.nn.rnn_cell.DropoutWrapper(bw_cell, output_keep_prob=self.dropout_keep_prob)
            (self.output_fw, self.output_bw), states = tf.nn.bidirectional_dynamic_rnn(cell_fw=fw_cell,
                                                                                       cell_bw=bw_cell,
                                                                                       inputs=self.embedding_words,
                                                                                       sequence_length=text_length,
                                                                                       dtype=tf.float32)

        with tf.name_scope("context"):
            shape = [tf.shape(self.output_fw)[0], 1, tf.shape(self.output_fw)[2]]
            self.context_left = tf.concat([tf.zeros(shape), self.output_fw[:, :-1]], axis=1, name="context_left")
            self.context_right = tf.concat([self.output_bw[:, 1:], tf.zeros(shape)], axis=1, name="context_right")

        with tf.name_scope("word-representation"):
            self.x = tf.concat([self.context_left, self.embedding_words, self.context_right], axis=2, name="x")
            embedding_size = 2*self.config.context_embedding_size + self.config.word_embedding_size

        # 2.1 apply nonlinearity
        with tf.name_scope("text-representation"):
            W2 = tf.Variable(tf.random_uniform([embedding_size, self.config.hidden_size], -1.0, 1.0), name="W2")
            b2 = tf.Variable(tf.constant(0.1, shape=[self.config.hidden_size]), name="b2")
            self.y2 = tf.tanh(tf.einsum('aij,jk->aik', self.x, W2) + b2)

        with tf.name_scope("text-representation"):
            W2 = tf.Variable(tf.random_uniform([embedding_size, self.config.hidden_size], -1.0, 1.0), name="W2")
            b2 = tf.Variable(tf.constant(0.1, shape=[self.config.hidden_size]), name="b2")
            self.y2 = tf.tanh(tf.einsum('aij,jk->aik', self.x, W2) + b2)

        # 3. max-pooling
        with tf.name_scope("max-pooling"):
            self.y3 = tf.reduce_max(self.y2, axis=1)  # shape:[None, hidden_size]

        # 4. classifier
        with tf.name_scope("output"):
            # inputs: A `Tensor` of shape `[batch_size, dim]`. The forward activations of the input network.
            W4 = tf.get_variable("W4", shape=[self.config.hidden_size, self.config.num_classes],
                                 initializer=tf.contrib.layers.xavier_initializer())
            b4 = tf.Variable(tf.constant(0.1, shape=[self.config.num_classes]), name="b4")
            l2_loss += tf.nn.l2_loss(W4)
            l2_loss += tf.nn.l2_loss(b4)
            self.logits = tf.nn.xw_plus_b(self.y3, W4, b4, name="logits")
            self.prediction = tf.argmax(tf.nn.softmax(self.logits), 1, name='prediction')

        # loss
        with tf.name_scope('loss'):
            losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, labels=self.input_y)
            self.loss = tf.reduce_mean(losses) + self.config.l2_reg_lambda * l2_loss

        # optimizer
        with tf.name_scope('optimizer'):
            optimizer = tf.train.AdamOptimizer(self.config.learning_rate)
            gradients, variables = zip(*optimizer.compute_gradients(self.loss))#计算变量梯度，得到梯度值、变量
            gradients, _ = tf.clip_by_global_norm(gradients, self.config.gradient_clip)
            #对g进行l2正则化计算，比较其与clip的值，如果l2后的值更大，让梯度*(clip/l2_g),得到新梯度
            self.optimizer = optimizer.apply_gradients(zip(gradients, variables), global_step=self.global_step)
            #global_step 自动+1

        with tf.name_scope('accuracy'):
            correct_prediction = tf.equal(self.prediction, tf.argmax(self.input_y, 1))
            self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')

    @staticmethod
    def _get_cell(hidden_size, rnn_type):
        if rnn_type == "Vanilla":
            return tf.nn.rnn_cell.BasicRNNCell(hidden_size)
        elif rnn_type == "LSTM":
            return tf.nn.rnn_cell.BasicLSTMCell(hidden_size)
        elif rnn_type == "GRU":
            return tf.nn.rnn_cell.GRUCell(hidden_size)
        else:
            print("ERROR: '" + rnn_type + "' is a wrong cell type !!!")
            return None

    # Length of the sequence data
    @staticmethod
    def _length(seq):
        relevant = tf.sign(tf.abs(seq))
        length = tf.reduce_sum(relevant, reduction_indices=1)
        length = tf.cast(length, tf.int32)
        return length

    # Extract the output of last cell of each sequence
    # Ex) The movie is good -> length = 4
    #     output = [ [1.314, -3.32, ..., 0.98]
    #                [0.287, -0.50, ..., 1.55]
    #                [2.194, -2.12, ..., 0.63]
    #                [1.938, -1.88, ..., 1.31]
    #                [  0.0,   0.0, ...,  0.0]
    #                ...
    #                [  0.0,   0.0, ...,  0.0] ]
    #     The output we need is 4th output of cell, so extract it.
    @staticmethod
    def last_relevant(seq, length):
        batch_size = tf.shape(seq)[0]
        max_length = int(seq.get_shape()[1])
        input_size = int(seq.get_shape()[2])
        index = tf.range(0, batch_size) * max_length + (length - 1)
        flat = tf.reshape(seq, [-1, input_size])
        return tf.gather(flat, index)
<file_sep>/0_Class/README.md
# NLP Class
<file_sep>/notes.md
# 笔记

## 1 模型分类(生成式与判别式)
![image](https://github.com/xuewengeophysics/xwStudyNLP/blob/master/images/%E6%A8%A1%E5%9E%8B%E5%88%86%E7%B1%BB(%E7%94%9F%E6%88%90%E5%BC%8F%E4%B8%8E%E5%88%A4%E5%88%AB%E5%BC%8F).png)
<file_sep>/elmo/README.md
# Embedding from Language Model

参考资料
==
1  [论文Deep contextualized word representations](https://arxiv.org/abs/1802.05365)<br>
<file_sep>/sentiment_analysis/README.md
# Sentiment Analysis

## 参考资料
1  https://github.com/sebastianruder/NLP-progress/blob/master/english/sentiment_analysis.md<br><br>
2  https://github.com/songyouwei/ABSA-PyTorch<br><br>
3  https://github.com/HSLCY/ABSA-BERT-pair<br><br>
4  https://github.com/AlexYangLi/ABSA_Keras<br><br>
<file_sep>/text_classification/BiLstmAttention/BiLstmAttentionModel.py
"""
参考资料：
[1]  https://github.com/SeoSangwoo/Attention-Based-BiLSTM-relation-extraction
"""

import tensorflow as tf

class BiLstmAttention(object):
    """
    Attention-Based Bidirectional Long Short-Term Memory Networks for Relation Classification
    1. Embeddding layer,
    2. Bi-LSTM layer,
    3. Attention layer,
    4. FC layer
    5. softmax
    """
    def __init__(self, config):
        self.config = config

        # placeholders for input output and dropout
        # input_x的维度是[batch_size, sequence_length]
        # input_y的维度是[batch_size, num_classes]
        self.input_x = tf.placeholder(tf.int32, shape=[None, self.config.sequence_length], name='input_x')
        self.input_y = tf.placeholder(tf.int32, shape=[None, self.config.num_classes], name='input_y')
        self.dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob')
        self.global_step = tf.Variable(0, trainable=False, name='global_step')

        # 定义l2损失
        l2_loss = tf.constant(0.0)
        text_length = self._length(self.input_x)

        # 1.get embedding of words in the sentence
        with tf.name_scope("embedding"):
            # self.embedding = tf.Variable(tf.random_uniform([self.config.vocab_size, self.config.word_embedding_size], -1.0, 1.0),
            #                              name="embedding", trainable=True)
            # self.embedding = tf.Variable(tf.truncated_normal([self.config.vocab_size, self.config.word_embedding_size],
            #                              stddev=1.0 / math.sqrt(self.config.word_embedding_size)),
            #                              name="embedding", trainable=True)
            self.embedding = tf.get_variable("embeddings", shape=[self.config.vocab_size, self.config.word_embedding_size],
                                             initializer=tf.constant_initializer(self.config.pre_training),
                                             trainable=True) # 是否使用预训练词向量，静态(False)或动态(默认True)
            # 张量维度为[None, sequence_length, word_embedding_size]
            self.embedding_words = tf.nn.embedding_lookup(self.embedding, self.input_x)

        # 2. BiLSTM layer to get high level features
        # 输出张量 x_i = [c_l(w_i); e(w_i); c_r(w_i)], shape:[None, sequence_length, embedding_size*3]
        with tf.name_scope("bi-rnn"):
            fw_cell = self._get_cell(self.config.context_embedding_size, self.config.rnn_type)
            fw_cell = tf.nn.rnn_cell.DropoutWrapper(fw_cell, output_keep_prob=self.dropout_keep_prob)
            bw_cell = self._get_cell(self.config.context_embedding_size, self.config.rnn_type)
            bw_cell = tf.nn.rnn_cell.DropoutWrapper(bw_cell, output_keep_prob=self.dropout_keep_prob)
            # 采用动态rnn，可以动态的输入序列的长度，若没有输入，则取序列的全长
            # outputs是一个元组(output_fw, output_bw)，其中两个元素的维度都是[batch_size, max_time, hidden_size]
            # fw和bw的hidden_size一样
            # states是最终的状态，二元组(state_fw, state_bw)，state_fw=[batch_size, s]，s是一个元组(h, c)
            self.outputs, states = tf.nn.bidirectional_dynamic_rnn(cell_fw=fw_cell, cell_bw=bw_cell,
                                                                   inputs=self.embedding_words,
                                                                   sequence_length=text_length,
                                                                   dtype=tf.float32)

            # 对outputs中的fw和bw的结果拼接 [batch_size, time_step, hidden_size * 2], 传入到下一层Bi-LSTM中
            self.rnn_outputs = tf.add(self.outputs[0], self.outputs[1])

        # 3. Attention layer
        # produce a weight vector,
        # and merge word-level features from each time step into a sentence-level feature vector,
        # by multiplying the weight vector
        with tf.variable_scope('attention'):
            self.attention, self.alphas = self._get_attention(self.rnn_outputs)

        # Dropout
        with tf.variable_scope('dropout'):
            self.h_drop = tf.nn.dropout(self.attention, self.dropout_keep_prob)

        # 4. classifier
        with tf.variable_scope('output'):
            # Fully connected layer
            self.logits = tf.layers.dense(self.h_drop, self.config.num_classes,
                                          kernel_initializer=tf.keras.initializers.glorot_normal())
            self.prediction = tf.argmax(self.logits, 1, name="prediction")

        # loss
        with tf.name_scope('loss'):
            losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, labels=self.input_y)
            self.loss = tf.reduce_mean(losses) + self.config.l2_reg_lambda * l2_loss

        # optimizer
        with tf.name_scope('optimizer'):
            optimizer = tf.train.AdamOptimizer(self.config.learning_rate)
            gradients, variables = zip(*optimizer.compute_gradients(self.loss))  # 计算变量梯度，得到梯度值、变量
            gradients, _ = tf.clip_by_global_norm(gradients, self.config.gradient_clip)
            # 对g进行l2正则化计算，比较其与clip的值，如果l2后的值更大，让梯度*(clip/l2_g),得到新梯度
            self.optimizer = optimizer.apply_gradients(zip(gradients, variables), global_step=self.global_step)
            # global_step 自动+1

        with tf.name_scope('accuracy'):
            correct_prediction = tf.equal(self.prediction, tf.argmax(self.input_y, 1))
            self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')

    def _get_attention(self, inputs):
        # Trainable parameters
        hidden_size = inputs.shape[2].value  # inputs的维度为[batch_size, time_step, hidden_size * 2]
        # omega_T的维度为[hidden_size * 2]
        omega_T = tf.get_variable("omega_T", [hidden_size], initializer=tf.keras.initializers.glorot_normal())

        with tf.name_scope('M'):
            # M的维度为[batch_size, time_step, hidden_size * 2]
            M = tf.tanh(inputs)

        # For each of the timestamps its vector of size A from `M` is reduced with `omega` vector
        vu = tf.tensordot(M, omega_T, axes=1, name='vu')  # [batch_size, time_step] shape
        alphas = tf.nn.softmax(vu, name='alphas')  # [batch_size, time_step] shape

        # Output of (Bi-)RNN is reduced with attention vector; the result has [batch_size, hidden_size * 2] shape
        output = tf.reduce_sum(inputs * tf.expand_dims(alphas, -1), 1)

        # Final output with tanh
        output = tf.tanh(output)

        return output, alphas

    @staticmethod
    def _get_cell(hidden_size, rnn_type):
        if rnn_type == "Vanilla":
            return tf.nn.rnn_cell.BasicRNNCell(hidden_size)
        elif rnn_type == "LSTM":
            return tf.nn.rnn_cell.BasicLSTMCell(hidden_size)
        elif rnn_type == "GRU":
            return tf.nn.rnn_cell.GRUCell(hidden_size)
        else:
            print("ERROR: '" + rnn_type + "' is a wrong cell type !!!")
            return None

    # Length of the sequence data
    @staticmethod
    def _length(seq):
        relevant = tf.sign(tf.abs(seq))
        length = tf.reduce_sum(relevant, reduction_indices=1)
        length = tf.cast(length, tf.int32)
        return length
<file_sep>/GPT/README.md
# Generative Pre-trained Transformer

参考资料
==
论文1 [GPT-Improving Language Understanding by Generative Pre-Training](https://s3-us-west-2.amazonaws.com/openai-assets/research-covers/language-unsupervised/language_understanding_paper.pdf)<br>
论文2 [Language Models are Unsupervised Multitask Learners](https://d4mucfpksywv.cloudfront.net/better-language-models/language-models.pdf)<br>

代码1 [openai/finetune-transformer-lm](https://github.com/openai/finetune-transformer-lm)<br>
代码2 [openai/gpt-2](https://github.com/openai/gpt-2)<br>
代码3 [ConnorJL/GPT2](https://github.com/ConnorJL/GPT2)<br>
<file_sep>/text_classification/TextCNN/config/Config.py
class TextcnnConfig(object):
    """Textcnn配置参数"""

    # 文本参数
    seq_length = 600  # max length of sentence
    num_classes = 10  # number of labels
    vocab_size = 8000  # number of vocabulary

    # 模型参数
    embedding_size = 100  # dimension of word embedding
    num_filters = 128  # number of convolution kernel
    kernel_size = [2, 3, 4]  # size of convolution kernel
    pre_trianing = None  # use vector_char trained by word2vec

    # 训练参数
    batch_size = 8  # 每批训练大小
    num_epochs = 5  # 总迭代轮次
    dropout_keep_prob = 0.5  # dropout保留比例
    learning_rate = 1e-3  # 学习率
    lr_decay = 0.9  # learning rate decay
    clip = 7.0   # gradient clipping threshold
    print_per_batch = 100  # 每多少轮输出一次结果
    save_per_batch = 10  # 每多少轮存入tensorboard
<file_sep>/text_classification/README.md
# Text Classification

0 参考资料
=
### 1、论文
1 [[<NAME> et al. 2019]Text Classification Algorithms: A Survey](https://arxiv.org/abs/1904.08067)<br>

### 2、代码
1 [徐亮brightmart/text_classification](https://github.com/brightmart/text_classification)<br>
2 [cjymz886/text-cnn](https://github.com/cjymz886/text-cnn)<br>
3 [文本分类实战--从TFIDF到深度学习CNN系列效果对比（附代码）](https://github.com/lc222/text_classification_AI100)<br>
4 [gaussic/CNN-RNN中文文本分类](https://github.com/gaussic/text-classification-cnn-rnn)<br>
5 [clayandgithub/基于cnn的中文文本分类算法](https://github.com/clayandgithub/zh_cnn_text_classify)<br>
6 [pengming617/text_classification](https://github.com/pengming617/text_classification)<br>
7 [roomylee/rcnn-text-classification](https://github.com/roomylee/rcnn-text-classification)<br>
8 [jiangxinyang227/textClassifier](https://github.com/jiangxinyang227/textClassifier)<br>

1 数据集
=
本实验使用THUCNews的一个子集进行训练与测试，数据集请自行到[THUCTC：一个高效的中文文本分类工具包](http://thuctc.thunlp.org/)下载，请遵循数据提供方的开源协议。<br>

文本类别涉及10个类别：categories = \['体育', '财经', '房产', '家居', '教育', '科技', '时尚', '时政', '游戏', '娱乐']，每个分类6500条数据；<br>
cnews.train.txt: 训练集(5000*10)<br>

cnews.val.txt: 验证集(500*10)<br>

cnews.test.txt: 测试集(1000*10)<br>

数据下载：训练数据以及训练好的词向量：<br>
链接: [https://pan.baidu.com/s/1DOgxlY42roBpOKAMKPPKWA](https://pan.baidu.com/s/1DOgxlY42roBpOKAMKPPKWA)<br>
密码: <PASSWORD><br><br>

2 预处理
=
本实验主要对训练文本进行分词处理，一来要分词训练词向量，二来输入模型的以词向量的形式；<br><br>
另外，词仅为中文或英文，词的长度大于1；<br><br>
处理的程序都放在loader.py文件中；<br><br>

3 运行步骤
=
python train_word2vec.py，对训练数据进行分词，利用Word2vec训练词向量(vector_word.txt)<br><br>
python text_train.py，进行训练模型<br><br>
python text_test.py，对模型进行测试<br><br>
python text_predict.py，提供模型的预测<br><br>


## TextCNN

## Transformer


<file_sep>/attention/README.md
# Attention

参考资料
==

### 1、论文

1  [[Bahdanau et al. ICLR2015]Neural Machine Translation by Jointly Learning to Align and Translate](https://arxiv.org/abs/1409.0473v7)<br>
2  [[Luong et al. EMNLP2015]Effective Approaches to Attention-based Neural Machine Translation](https://aclweb.org/anthology/D15-1166)<br>
3  [[Kel<NAME> et al. ICML2015]Show, Attend and Tell: Neural Image Caption Generation with Visual Attention](https://arxiv.org/abs/1502.03044)<br>
4  [[<NAME> et al. 2016]Hierarchical Attention Networks for Document Classification](https://www.cs.cmu.edu/~./hovy/papers/16HLT-hierarchical-attention-networks.pdf)<br>
5  [[Vaswani et al. ICCL2017]Attention Is All You Need](https://arxiv.org/abs/1706.03762)<br>
6  [[Y<NAME> et al. 2016]Attention-based LSTM for Aspect-level Sentiment Classification](https://aclweb.org/anthology/D16-1058)<br>
7  [[Zhouhan Lin et al. 2017]A Structured Self-attentive Sentence Embedding](https://arxiv.org/abs/1703.03130)<br>

### 2、博客

1  [Nir Arbel/Attention in RNNs](https://medium.com/datadriveninvestor/attention-in-rnns-321fbcd64f05)  [对应翻译/AI公园/戎怀阳/使用详细的例子来理解RNN中的注意力机制](https://mp.weixin.qq.com/s/j21govyAwBQehmJYmSsYIw)<br>

### 3、代码

<file_sep>/probabilistic_graphical_models/README.md
# 概率图模型

0 参考资料
=

crf
=
1 [苏剑林/简明条件随机场CRF介绍（附带纯Keras实现）](https://kexue.fm/archives/5542)<br>

<file_sep>/seq2seq/README.md
# sequence to sequence

## 参考资料

[[1] <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., & <NAME>. (2014). Learning phrase representations using RNN encoder-decoder for statistical machine translation.](https://arxiv.org/abs/1406.1078.pdf)<br>

[[2] <NAME>., <NAME>., & <NAME>. (2014). Sequence to sequence learning with neural networks.](https://arxiv.org/pdf/1409.3215.pdf)<br>

博客1：[完全图解RNN、RNN变体、Seq2Seq、Attention机制](https://www.leiphone.com/news/201709/8tDpwklrKubaecTa.html)

博客2：[自然语言处理中的Attention机制总结](https://blog.csdn.net/hahajinbu/article/details/81940355)
<file_sep>/text_classification/TextRNN/TextRnnModel.py
import tensorflow as tf

class TextRnn(object):
    """
    RNN for text classification
    """
    def __init__(self, config):
        self.config = config

        # placeholders for input output and dropout
        # input_x的维度是[batch_size, sequence_length]
        # input_y的维度是[batch_size, num_classes]
        self.input_x = tf.placeholder(tf.int32, shape=[None, self.config.sequence_length], name='input_x')
        self.input_y = tf.placeholder(tf.int32, shape=[None, self.config.num_classes], name='input_y')
        self.seq_length = tf.placeholder(tf.int32, shape=[None], name='seq_length')
        self.dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob')
        self.global_step = tf.Variable(0, trainable=False, name='global_step')

        # word embedding
        with tf.name_scope("embedding"):
            # self.embedding = tf.Variable(tf.random_uniform([self.config.vocab_size, self.config.embedding_size], -1.0, 1.0),
            #                              name="embedding", trainable=True)
            # self.embedding = tf.Variable(tf.truncated_normal([self.config.vocab_size, self.config.embedding_size],
            #                              stddev=1.0 / math.sqrt(self.config.embedding_size)),
            #                              name="embedding", trainable=True)
            self.embedding = tf.get_variable("embeddings", shape=[self.config.vocab_size, self.config.embedding_size],
                                             initializer=tf.constant_initializer(self.config.pre_training),
                                             trainable=True) # 是否使用预训练词向量，静态(False)或动态(默认True)
            # 张量维度为[None, sequence_length, embedding_size]
            self.embedding_inputs = tf.nn.embedding_lookup(self.embedding, self.input_x)

        # RNN layer
        with tf.name_scope("rnn_layer"):
            cells = []
            for _ in range(self.config.hidden_layer_num):
                lstm_cell = tf.contrib.rnn.BasicLSTMCell(self.config.hidden_size, forget_bias=0.0, state_is_tuple=True)
                lstm_cell = tf.contrib.rnn.DropoutWrapper(lstm_cell, output_keep_prob=self.dropout_keep_prob)
                cells.append(lstm_cell)
            cell = tf.nn.rnn_cell.MultiRNNCell(cells, state_is_tuple=True)
            self.embedding_inputs = tf.nn.dropout(self.embedding_inputs, self.dropout_keep_prob)
            # sequence_length: （可选）大小为[batch_size],数据的类型是int32/int64向量
            outputs, states = tf.nn.dynamic_rnn(cell, self.embedding_inputs, dtype=tf.float32,
                                                sequence_length=self.seq_length)
            # outputs:[batch_size, sequence_length, hidden_size]
            self.outputs = tf.reduce_sum(outputs, axis=1)

        # dropout
        with tf.name_scope('dropout'):
            self.outputs_dropout = tf.nn.dropout(self.outputs, keep_prob=self.dropout_keep_prob)

        # classifier
        with tf.name_scope('output'):
            W = tf.Variable(tf.truncated_normal([self.config.hidden_size, self.config.num_classes], stddev=0.1), name='W')
            b = tf.Variable(tf.constant(0.1, shape=[self.config.num_classes]), name='b')
            self.logits = tf.matmul(self.outputs_dropout, W) + b
            self.predict = tf.argmax(tf.nn.softmax(self.logits), 1, name='predict')

        # loss
        with tf.name_scope('loss'):
            losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, labels=self.input_y)
            self.loss = tf.reduce_mean(losses)

        # optimizer
        with tf.name_scope('optimizer'):
            optimizer = tf.train.AdamOptimizer(self.config.learning_rate)
            gradients, variables = zip(*optimizer.compute_gradients(self.loss))#计算变量梯度，得到梯度值、变量
            gradients, _ = tf.clip_by_global_norm(gradients, self.config.gradient_clip)
            #对g进行l2正则化计算，比较其与clip的值，如果l2后的值更大，让梯度*(clip/l2_g),得到新梯度
            self.optimizer = optimizer.apply_gradients(zip(gradients, variables), global_step=self.global_step)
            #global_step 自动+1

        with tf.name_scope('accuracy'):
            correct_prediction = tf.equal(self.predict, tf.argmax(self.input_y, 1))
            self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')
<file_sep>/word2vec/README.md
# word2vec

## 参考资料

源码1：[AimeeLee77/wiki_zh_word2vec](https://github.com/AimeeLee77/wiki_zh_word2vec)
<file_sep>/scrapy/README.md
# 爬虫

## 使用Scrapy实现爬虫的主要步骤：

1. 创建一个Scrapy项目。

    scrapy startproject tianya_textclassify
    
2. 定义提取的Item。在items.py文件定义想要提取的实体：

    class TianyaTextclassifyItem(scrapy.Item):
    
        label   = scrapy.Field() # 文档标签
        
        content = scrapy.Field() # 文档内容
        
3. 编写爬取网站的 spider 并提取 Item。创建并修改spiders中蜘蛛文件tianya_ly.py：

    scrapy genspider tianya_ly tianya.cn
   
4. 编写 Item Pipeline 来存储提取到的Item(即数据)。在pipelines.py中定义函数用于存储数据：

    def process_item(self, item, spider):
    
        with open('train_ly.txt', 'a', encoding='utf8') as fw:
        
            fw.write(item['label'] + "," + item['content'] + '\n')
            
        return item
5. 创建执行程序的文件main.py。有两种执行方式：

    (1)：用cmdline执行cmd命令
    
    from scrapy import cmdline
    
    cmdline.execute("scrapy crawl tianya_ly".split())
    
    (2)：常用方式
    
    from scrapy.cmdline import execute
    
    execute(["scrapy", "crawl", "tianya_ly"])
    

## scrapy工作原理图
![image](https://github.com/xuewengeophysics/xwStudyNLP/blob/master/scrapy/images/%E7%88%AC%E8%99%AB.jpg)

## 参考资料
[1] https://scrapy-chs.readthedocs.io/zh_CN/latest/intro/overview.html
<file_sep>/text_classification/BiLstmAttention/README.md
# Attention-Based Bidirectional Long Short-Term Memory Networks for Text Classification

参考资料
=
1  [论文[<NAME> et al. 2016]Attention-Based Bidirectional Long Short-Term Memory Networks for Relation Classification](https://www.aclweb.org/anthology/P16-2034)<br>
2  [代码SeoSangwoo/Attention-Based-BiLSTM-relation-extraction](https://github.com/SeoSangwoo/Attention-Based-BiLSTM-relation-extraction)<br>
<file_sep>/text_classification/Transformer/README.md
# Transformer

## 参考资料

论文：[Attention Is All You Need](https://arxiv.org/pdf/1706.03762.pdf)

源码1：[brightmart/text_classification/a07_Transformer](https://github.com/brightmart/text_classification/tree/master/a07_Transformer)

源码2：[pengming617/text_classification/model/transformer](https://github.com/pengming617/text_classification/tree/master/model/transformer)

博客1：[文本分类实战之Transformer模型](https://www.cnblogs.com/jiangxinyang/p/10210813.html)
博客1对应源码：[jiangxinyang227/textClassifier/Transformer](https://github.com/jiangxinyang227/textClassifier/tree/master/Transformer)
<file_sep>/0_Class/CMU CS 11-747/README.md
# Study CMU CS 11-747 nn4nlp2019

## 学习资料

1 [课程官网Neural Networks for NLP](http://www.phontron.com/class/nn4nlp2019/)<br>
2 [课程代码neubig/nn4nlp-code](https://github.com/neubig/nn4nlp-code)<br>
3 [课程翻译AI研习社](https://ai.yanxishe.com/page/groupDetail/33)<br>
4 [课程第9课讨论的代码xnmt](https://github.com/neulab/xnmt)<br>
5 [课程第9课讨论的代码compare-mt](https://github.com/neulab/compare-mt)<br>
<file_sep>/BERT/README.md
# Bidirectional Encoder Representations from Transformers

参考资料
==
1  [论文BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805)<br>
