自然语言处理应用

# Tensorflow实现用于多标签文本分类的TextCNN结构
def TextCNN(seq_len, token_num, embed_dim, out_dim, model_img_path=None, embed_matrix=None):
    """
    TextCNN模型
    1. embedding layers
    2. convolution layer
    3. max-pooling
    4. softmax layer
    """
    x_input = tf.keras.Input(shape=(max_seq_len,))
    if embedd_matrix is None:
        x_emb = tf.keras.layers.Embedding(input_dim=token_num,
                                          output_dim=embed_dim,
                                          input_length=seq_len)(x_input)
    else:  # 加载预训练好的词向量矩阵
        x_emb = tf.keras.layers.Embedding(input_dim=token_num,
                                          output_dim=embed_dim,
                                          input_length=seq_len,
                                          weights=[embed_matrix],
                                          trainable=True)(x_input)

    # x_embed shape: (batch_size, seq_len, token_num)
    pool_output = []
    kernel_sizes = [2, 3, 4]  # 设置多种卷积核
    for kernel_size in kernel_sizes:
        # Conv1D input shape: 形如(samples, steps, input_dim)的3D张量
        # Conv1D output shape: 形如(samples，new_steps，nb_filter)的3D张量
        c = tf.keras.layers.kerasConv1D(filters=100, kernel_size=kernel_size, strides=1)(x_emb)
        p = tf.keras.layers.MaxPool1D(pool_size=int(c.shape[1]))(c)
        pool_output.append(p)

    pool_output = tf.keras.layers.concatenate([p for p in pool_output])  # 将多个池化结果进行拼接
    x_flatten = tf.keras.layers.Flatten()(pool_output)
    # output_dim个sigmoid输出，是多标签分类的实现方式，实质上是对每个标签的二分类
    y = tf.keras.layers.Dense(output_dim, activation='sigmoid')(x_flatten)

    model = tf.keras.Model([x_input], outputs=[y])
    model.summary()
    return model

# 使用Tensorflow序列方式构建FastText文本多分类模型
def build_fasttext_model(vocab, embed, length, num_hidden, num_classes):
    fasttext = tf.keras.Sequential()
    fasttext.add(tf.keras.layers.Embedding(input_dim=vocab,
                                           output_dim=embed,
                                           input_length=length,
                                           mask_zero=True))
    fasttext.add(tf.keras.layers.GlobalAveragePooling1D())
    fasttext.add(tf.keras.layers.Dense(num_hidden, activation='relu'))
    fasttext.add(tf.keras.layers.Dense(num_classes, activation='softmax'))
    fasttext.summary()
    return fasttext

# 使用Tensorflow序列方式构建RNN文本多分类模型
def build_rnn_model(vocab, embed, length, num_hidden, num_classes):
    rnn = tf.keras.Sequential()
    rnn.add(tf.keras.layers.Embedding(input_dim=vocab,
                                      output_dim=embed,
                                      input_length=length,
                                      mask_zero=True))
    rnn.add(tf.keras.layers.SimpleRNN(num_hidden, activation='relu'))  # 可替换为LSTM/GRU
    rnn.add(tf.keras.layers.Dense(num_hidden, activation='relu'))
    rnn.add(tf.keras.layers.Dense(num_classes, activation='softmax'))
    rnn.summary()
    return rnn

# 微调transformers中的BERT模型进行中文文本分类(新闻标题分类)
import os
import tensorflow as tf
from transformers import BertTokenizer, TFBertForSequenceClassification
os.environ['CUDA_VISIBLE_DEVICES'] = '0'

# 设置模型和文件路径，运行前需要将模型和文件导入当前文件夹
bert_model_path = 'bert-chinese'
bert_fine_tuned_path = 'bert-cls'
train_data_path = 'bert_classification_train.txt'
dev_data_path = 'bert_classification_dev.txt'
test_data_path = 'bert_classification_test.txt'

gpus = tf.config.experimental.list_physical_devices('GPU')
print(gpus)
# 设置按需申请，multi-GPU需要for一下
tf.config.experimental.set_memory_growth(gpus[0], True)

# mapping
y_map = {'财经': 0, '彩票': 1, '房产': 2, '股票': 3, '家居': 4, '教育': 5, '科技': 6, 
         '社会': 7, '时尚': 8, '时政': 9, '体育': 10, '星座': 11, '游戏': 12, '娱乐': 13}
y_map_reverse = {y_map[k] : k for k in y_map.keys()}
print(y_map, y_map_reverse)

def load_model(bert_model_path):
    tokenizer = BertTokenizer.from_pretrained(bert_model_path)
    model = TFBertForSequenceClassification.from_pretrained(bert_model_path, num_labels=14)  # 类别数目
    tokenizer.pad_token = '[PAD]'
    return tokenizer, model


def build_dataset():
    # 读取文件
    X_train, X_dev, y_train, y_dev = [], [], [], []
    with open(train_data_path, 'r', encoding='utf-8') as f:
        lines = f.read().split('\n')
    for i in range(len(lines)):
        text, label = lines[i].split('\t')
        X_train.append(text)
        y_train.append(y_map[label])
    with open(dev_data_path, 'r', encoding='utf-8') as f:
        lines = f.read().split('\n')
    for i in range(len(lines)):
        text, label = lines[i].split('\t')
        X_dev.append(text)
        y_dev.append(y_map[label])
    return X_train, X_dev, y_train, y_dev


def fine_tuning(bert_model_path, X_train, y_train, X_dev, y_dev, model_save_path, epochs=20, batch_size=64):
    tokenizer, model = load_model(bert_model_path)
    print(len(X_train), len(y_train), len(X_dev), len(y_dev))
    print('encoding train set and dev set by tokenizer')
    train_encodings = tokenizer(X_train, truncation=True, padding=True, max_length=32, return_tensors='tf')
    dev_encodings = tokenizer(X_dev, truncation=True, padding=True, max_length=32, return_tensors='tf')
    print(train_encodings.keys())  # ['input_ids', 'token_type_ids', 'attention_mask']

    train_tensor_inputs = tf.convert_to_tensor(train_encodings['input_ids'])
    train_tensor_labels = tf.convert_to_tensor(y_train)
    train_tensor_mask = tf.convert_to_tensor(train_encodings['attention_mask'])
    train_ds = tf.data.Dataset.from_tensor_slices(
        (
            {'input_ids': train_tensor_inputs, 'attention_mask': train_tensor_mask},
            train_tensor_labels,
        )
    )

    dev_tensor_inputs = tf.convert_to_tensor(dev_encodings['input_ids'])
    dev_tensor_labels = tf.convert_to_tensor(y_dev)
    dev_tensor_mask = tf.convert_to_tensor(dev_encodings['attention_mask'])
    dev_ds = tf.data.Dataset.from_tensor_slices(
        (
            {'input_ids': dev_tensor_inputs, 'attention_mask': dev_tensor_mask},
            dev_tensor_labels,
        )
    )

    train_ds = train_ds.shuffle(len(X_train)).batch(batch_size, drop_remainder=True)
    dev_ds = dev_ds.batch(batch_size)

    print(train_ds)
    print(dev_ds)

    optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
    loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
    model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
    model.summary()

    # train and save the model
    for i in range(epochs):
        print('training epoch', i + 1)
        hist = model.fit(train_ds, epochs=1, validation_data=dev_ds)
        model.save_pretrained(model_save_path + '_' + str(i + 1) + '_epoch')


def inference(bert_model_path, result_path):
    tokenizer, model = load_model(bert_model_path)
    with open(test_data_path, 'r', encoding='utf-8') as f:
        test_data = f.read().split('\n')[:-1]
    print(len(test_data))
    test_encodings = tokenizer(test_data, truncation=True, padding=True, max_length=32, return_tensors='tf')
    test_tensor_inputs = tf.convert_to_tensor(test_encodings['input_ids'])
    test_tensor_mask = tf.convert_to_tensor(test_encodings['attention_mask'])
    test_ds = tf.data.Dataset.from_tensor_slices(({'input_ids': test_tensor_inputs, 'attention_mask': test_tensor_mask}))
    test_ds = test_ds.batch(128)
    print(test_ds)

    with open(result_path, 'a', encoding='utf-8') as f:
        for i, batch_data in enumerate(test_ds):
            if i % 10 == 0:
                print(i)
            out_dist = model(batch_data)[0]
            out_softmax = tf.nn.softmax(out_dist, axis=1)
            preds = list(tf.math.argmax(out_softmax, axis=1).numpy())
            for pred in preds:
                label = y_map_reverse[pred]
                f.write(label + '\n')


if __name__ == '__main__':
    X_train, X_dev, y_train, y_dev = build_dataset()
    # fine_tuning(bert_model_path, X_train, y_train, X_dev, y_dev, bert_fine_tuned_path)
    inference(bert_fine_tuned_path + '_3_epoch', 'result.txt')

# 利用textrank4zh中的TextRank4Sentence实现中文抽提式文本摘要
from textrank4zh import TextRank4Sentence

results = []  # results用于保存摘要结果
tr4s = TextRank4Sentence()  # TextRank4Sentence对象

for i in range(len(texts)):  # texts为文档集合
    text = texts[i]
    text_divide = re.split('[。]', text)  # 每个文档可以切分为多个句子，tr4s模型负责选出重要的句子完成摘要

    sort_dict = {j: text_divide[j] for j in range(len(text_divide))}  # 句子按照原顺序排序的字典(次序: 句子)
    reverse_sort_dict = {text_divide[j]: j for j in range(len(text_divide))}  # 反向排序字典(句子: 次序)

    # 利用tr4s完成摘要
    tr4s.analyze(text=text, lower=True, source='all_filters')
    result = tr4s.get_key_sentences(num=3, sentence_min_len=3)  # 选取长度大于等于3的最重要的3条句子
    result_list = []
    for item in result:
        result_list.append(item.sentence)

    # 对句子顺寻按照原文档顺序重新排序
    for j in range(len(result_list)):
        result_list[j] = reverse_sort_dict[result_list[j]]  # 变成顺序下标
    result_list.sort()  # 对顺序下标进行排序
    for j in range(len(result_list)):
        result_list[j] = sort_dict[result_list[j]]  # 利用词典恢复成句子

    results.append(result_list)

# 微调transformers中的GPT-2模型进行文本摘要(汽车大师对话摘要)
import os
import pandas as pd
import tensorflow as tf
from transformers import BertTokenizer, TFGPT2LMHeadModel
os.environ['CUDA_VISIBLE_DEVICES'] = '0'

# 设置模型和文件路径，运行前需要将模型和文件导入当前文件夹
gpt2_model_path = 'gpt2-chinese'
gpt2_fine_tuned_path = 'gpt2-sum'
train_data_path = 'gpt2_summarization_train.csv'
test_data_path = 'gpt2_summarization_test.csv'

gpus = tf.config.experimental.list_physical_devices('GPU')
print(gpus)
# 设置按需申请，multi-GPU需要for一下
tf.config.experimental.set_memory_growth(gpus[0], True)


def load_model(gpt2_model_path):
    tokenizer = BertTokenizer.from_pretrained(gpt2_model_path)
    tokenizer.pad_token = '[PAD]'
    # 设置eos_token_id用于early stopping
    model = TFGPT2LMHeadModel.from_pretrained(gpt2_model_path, pad_token_id=tokenizer.pad_token_id, eos_token_id = tokenizer.pad_token_id)

    return tokenizer, model


def build_dataset():
    # 读取原始csv文件
    train_df = pd.read_csv(train_data_path, engine='python', encoding='utf-8')  # 必须utf-8
    test_df = pd.read_csv(test_data_path, engine='python', encoding='utf-8')  # 必须utf-8
    # 空值去除
    train_df.dropna(subset=['Question', 'Dialogue', 'Report'], how='any', inplace=True)
    test_df.dropna(subset=['Question', 'Dialogue'], how='any', inplace=True)

    train_question_set = list(train_df['Question'])
    train_dialogue_set = list(train_df['Dialogue'])
    train_report_set = list(train_df['Report'])
    test_question_set = list(test_df['Question'])
    test_dialogue_set = list(test_df['Dialogue'])

    train_dataset = []
    test_dataset = []
    for i in range(len(train_question_set)):
        context = train_question_set[i] + ' [CLS] ' + train_dialogue_set[i]
        if len(context) > 480:
            context = context[:480]
        data = context + ' [SEP] ' + train_report_set[i]
        train_dataset.append(data)
    for i in range(len(test_question_set)):
        context = test_question_set[i] + ' [CLS] ' + test_dialogue_set[i]
        if len(context) > 480:
            context = context[:480]
        data = context + ' [SEP]'
        test_dataset.append(data)

    return train_dataset, test_dataset


def fine_tuning(gpt2_model_path, train_set, model_save_path, epochs=20, batch_size=4):
    tokenizer, model = load_model(gpt2_model_path)
    print(len(train_set))
    print('encoding train set by tokenizer')
    train_encodings = tokenizer(train_set, truncation=True, padding=True, max_length=512, return_tensors='tf', add_special_tokens=False)

    train_encodings['labels'] = [x[1:] for x in train_encodings['input_ids']]
    train_encodings['input_ids'] = [x[:-1] for x in train_encodings['input_ids']]
    train_encodings['attention_mask'] = [x[:-1] for x in train_encodings['attention_mask']]

    train_tensor_inputs = tf.convert_to_tensor(train_encodings['input_ids'])
    train_tensor_labels = tf.convert_to_tensor(train_encodings['labels'])
    train_tensor_mask = tf.convert_to_tensor(train_encodings['attention_mask'])
    train_ds = tf.data.Dataset.from_tensor_slices(
        (
            {'input_ids': train_tensor_inputs, 'attention_mask': train_tensor_mask},
            train_tensor_labels,
        )
    )

    train_ds = train_ds.shuffle(len(train_set)).batch(batch_size, drop_remainder=True)
    print(train_ds)

    optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
    loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
    model.compile(optimizer=optimizer, loss=[loss, *[None] * model.config.n_layer])
    model.summary()

    # 训练并保存模型
    for i in range(epochs):
        print('training epoch', i + 1)
        model.fit(train_ds, epochs=1)
        model.save_pretrained(model_save_path + '_' + str(i + 1) + '_epoch')


def generate(gpt2_model_path, test_set, result_path):
    tokenizer, model = load_model(gpt2_model_path)

    for i in range(len(test_set)):
        sentence = test_set[i]
        input_ids = tokenizer.encode(sentence, return_tensors='tf', add_special_tokens=False)

        # sampling (文本摘要使用sampling会导致结果不准确)
        # outputs = model.generate(input_ids, do_sample=True, max_length=min(512, input_ids.shape[1] + 256), top_k=100, top_p=0.98, early_stopping=True, no_repeat_ngram_size=2, num_return_sequences=10)
        # greedy search
        # outputs = model.generate(input_ids, max_length=min(512, input_ids.shape[1] + 256), early_stopping=True)
        # beam search (用于摘要效果最佳)
        outputs = model.generate(input_ids, max_length=min(512, input_ids.shape[1] + 256), num_beams=5, early_stopping=True)

        output_str = str(tokenizer.decode(outputs[0], skip_special_tokens=False))
        output_str = ''.join(output_str.split(' '))
        print(i + 1, output_str)

        # 保存结果文件
        with open(result_path, 'a', encoding='utf-8') as f:
             f.write('Q' + str(i + 1) + ',' + output_str.split('[SEP]')[-1] + '\n')


if __name__ == '__main__':
    train_set, test_set = build_dataset()

    # fine_tuning(gpt2_model_path, train_set, gpt2_fine_tuned_path + '_1_epoch')
    # 指明使用训练了几轮的模型进行inference，此处仅使用了训练一轮的GPT-2
    generate(gpt2_fine_tuned_path + '_1_epoch', test_set, 'result.csv')