说明
之前写过用lstm模型做的文本分类，但是代码结构非常混乱。读过Bert源码后，决定模仿Bert的结构，自己重新写一遍使用lstm模型的代码。只作为熟悉tensorflow各个api与一个比较清楚的NLP模型结构的练手用，不求更高的准确率。
使用包含10个商品类别，60000+数据的，已标注正负情感的商品评论数据作为训练语料。原文件为csv格式，包含3个字段：cat（类别）、label（正负情感）、review（实际的评论文字）。其既可作为正负情感分类的二分类训练语料，也可以（勉强）作为商品类别分类的10分类训练语料。
数据取自https://github.com/SophonPlus/ChineseNlpCorpus，非常感谢
已按大约8：2的比例得到了训练集与测试集并将其转为了tsv文件。
使用结巴分词作为切词工具。
一、得到词典txt文件
由于我一直没有找到合适的中文词典文件，Bert中的词典文件又是以字作为最小单位的，故这里暂时只是对待训练的语料做切词、去重的处理，得到一个比较小的字典文件。
def create_vocab():
raw_df = pd.read_csv(RAW_DATA) # 读原始文件为dataframe
# 热水器有一条数据有问题，不要热水器的数据
raw_df = raw_df[raw_df.cat != '热水器']
raw_document = raw_df['review'].tolist() # 原始语料（list形式）
# 加载停用词列表
# with open(STOPWORDS, 'r', encoding='utf-8') as s:
# stopwords = [word.strip() for word in s.readlines()]
document_words = [] # 原始语料完成切词
for sentence in raw_document:
cut_sentence = [word for word in jieba.lcut(sentence)]
document_words.extend(cut_sentence)
vocab_list = set(document_words)
with open(VOCAB, 'w', encoding='utf-8') as f:
f.write('[PAD]' + '\n')
f.write('[UNK]' + '\n')
for vocab in vocab_list:
f.write(vocab + '\n')
二、Tokenization
这一块完全基于Bert源码，做了非常多的精简。只满足：to_unicode、读取词典、切词、词语转id、id转词语的基本功能。没什么好说的。
import collections
import tensorflow as tf
import jieba
def convert_to_unicode(text):
"""Converts `text` to Unicode (if it's not already), assuming utf-8 input."""
if isinstance(text, str):
return text
elif isinstance(text, bytes):
return text.decode("utf-8", "ignore")
else:
raise ValueError("Unsupported string type: %s" % (type(text)))
# 将词典中的词构成(词，index)的collections.OrderedDict形式
def load_vocab(vocab_file):
"""Loads a vocabulary file into a dictionary."""
vocab = collections.OrderedDict()
index = 0
with tf.gfile.GFile(vocab_file, "r") as reader:
while True:
token = convert_to_unicode(reader.readline())
if not token:
break
token = token.strip()
vocab[token] = index
index += 1
return vocab
def convert_by_vocab(vocab, items):
"""Converts a sequence of [tokens|ids] using the vocab."""
output = []
for item in items:
output.append(vocab.get(item, vocab['[UNK]']))
return output
class FullTokenizer(object):
"""Runs end-to-end tokenziation."""
def __init__(self, vocab_file):
# 根据vocab文件，得到形如(词，index)的字典
self.vocab = load_vocab(vocab_file)
# 变成 index: 词 的形式
self.inv_vocab = {v: k for k, v in self.vocab.items()}
# 将句子变成词列表
@staticmethod
def tokenize(text):
split_tokens = jieba.lcut(text)
return split_tokens
def convert_tokens_to_ids(self, tokens):
return convert_by_vocab(self.vocab, tokens)
def convert_ids_to_tokens(self, ids):
return convert_by_vocab(self.inv_vocab, ids)
三、建立模型结构
模型的mode参数可取train、eval、predict三类，取eval时只返回cost与accuracy，取predict时只返回logits。别的不用多说了。
import tensorflow as tf
import json
import six
class LstmConfig(object):
def __init__(self,
vocab_size, # 词典中的词数
hidden_size=128,
keep_prob=0.9,
embedding_keep_prob=0.9, # 词向量不被dropout的比例
max_grad_norm=5,
num_of_classes=2, # 分类数
num_of_layers=2, # lstm网络层数
initializer_range=0.02): # 初始化范围
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.embedding_keep_prob = embedding_keep_prob
self.max_grad_norm = max_grad_norm
self.num_of_classes = num_of_classes
self.num_of_layers = num_of_layers
self.initializer_range = initializer_range
@classmethod
def from_dict(cls, json_object):
"""Constructs a `BertConfig` from a Python dictionary of parameters."""
config = LstmConfig(vocab_size=None)
for (key, value) in six.iteritems(json_object):
config.__dict__[key] = value
return config
@classmethod
def from_json_file(cls, json_file):
"""Constructs a `BertConfig` from a json file of parameters."""
with tf.gfile.GFile(json_file, "r") as reader:
text = reader.read()
return cls.from_dict(json.loads(text))
# 双向LSTM网络模型
class LstmModel(object):
# 构建网格结构
def __init__(self, config, mode):
self.config = config
self.embedding_keep_prob = config.embedding_keep_prob
self.mode = mode
output_keep_prob = config.keep_prob if mode == 'train' else 1.0
# 词向量
self.word_embedding = tf.get_variable('word_emb', shape=[config.vocab_size, config.hidden_size])
# lstm网络结构
# 前向网络变量
lstm_cells_fw = [tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.BasicLSTMCell(config.hidden_size),
output_keep_prob=output_keep_prob)
for _ in range(config.num_of_layers)]
self.lstm_fw = tf.nn.rnn_cell.MultiRNNCell(lstm_cells_fw)
# 反向网络
lstm_cells_bw = [tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.BasicLSTMCell(config.hidden_size),
output_keep_prob=output_keep_prob)
for _ in range(config.num_of_layers)]
self.lstm_bw = tf.nn.rnn_cell.MultiRNNCell(lstm_cells_bw)
# Softmax层变量
self.weight = tf.get_variable('weight', [config.hidden_size * 2, config.num_of_classes])
self.bias = tf.get_variable('bias', [config.num_of_classes])
# 定义模型的前向计算图
def forward(self, src_input, src_size, label):
# 将输入的序号化单词转成词向量
inputs = tf.nn.embedding_lookup(self.word_embedding, src_input)
if self.mode == 'train':
inputs = tf.nn.dropout(inputs, self.embedding_keep_prob)
# LSTM网络计算
with tf.variable_scope('lstm'):
outputs, states = tf.nn.bidirectional_dynamic_rnn(self.lstm_fw,
self.lstm_bw,
inputs,
dtype=tf.float32,
sequence_length=src_size)
final_outputs = tf.concat(outputs, 2)
final_outputs = final_outputs[:, -1, :]
# 取平均值
# final_outputs = tf.reduce_mean(tf.concat(outputs, 2), 1)
# 全连接层计算
with tf.variable_scope('fc'):
logits = tf.matmul(final_outputs, self.weight) + self.bias
if self.mode == 'predict':
return logits
# 损失函数
with tf.variable_scope('loss'):
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=label,
logits=logits)
cost = tf.reduce_mean(loss)
# 准确率
with tf.variable_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(logits, 1), label)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
if self.mode == 'eval':
return cost, accuracy
# 定义反向操作
trainable_variables = tf.trainable_variables()
# 控制梯度大小，定义优化方法和训练步骤
grads = tf.gradients(cost, trainable_variables)
grads, _ = tf.clip_by_global_norm(grads, self.config.max_grad_norm)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.config.learning_rate)
train_op = optimizer.apply_gradients(zip(grads, trainable_variables))
return logits, cost, accuracy, train_op
四、分类主程序
我命名为run_classifier，完全照搬Bert。
首先是tf.flag，用于导入参数
flags = tf.flags
FLAGS = flags.FLAGS
flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.")
flags.DEFINE_integer("eval_batch_size", 8, "Total batch size for eval.")
flags.DEFINE_integer("predict_batch_size", 8, "Total batch size for predict.")
flags.DEFINE_integer("num_train_epochs", 4, "Total epoches for train.")
flags.DEFINE_string(
"data_dir", "E:/NLP/NLP_Deep_Learning_Summary/datasets",
"The input data dir. Should contain the .tsv files (or other data files) for the task.")
flags.DEFINE_string("init_checkpoint", None, "Initial checkpoint")
flags.DEFINE_string("vocab_file", "./vocab.txt", "The vocabulary file.")
flags.DEFINE_string("output_file", "./model1", "The output file for trained model.")
flags.DEFINE_bool("do_train", True, "Whether to run training.")
flags.DEFINE_bool("do_eval", True, "Whether to run eval on the dev set.")
flags.DEFINE_bool("do_predict", False, "Whether to run the model in inference mode on the test set.")
定义Example、Feature与DataProcessor类
为了从训练/测试数据中获得tfrecord文件，需要做Example与Feature类的处理。DataProcessor与Bert源码几乎相同，功能为从tsv文件中得到example类的数据。
tsv文件的各个字段为：index（id）、category（商品类别）、polarity（情感，0或1）、text（原始文字）。
class InputExample(object):
"""A single training/test example for simple sequence classification."""
def __init__(self, guid, text, label=None):
"""Constructs a InputExample."""
self.guid = guid
self.text = text
self.label = label
由于使用的是lstm，其特征包含：每条语料原始的词id、原始的长度（即词数）与原始的分类标签。
class InputFeatures(object):
def __init__(self, input_ids, input_size, label):
self.input_ids = input_ids
self.input_size = input_size
self.label = label
class DataProcessor(object):
def get_train_examples(self, data_dir):
lines = self._read_tsv(os.path.join(data_dir, "online_shopping_train.tsv"))
return self._create_examples(lines, 'train')
def get_dev_examples(self, data_dir):
lines = self._read_tsv(os.path.join(data_dir, "online_shopping_dev.tsv"))
return self._create_examples(lines, 'dev')
def get_test_examples(self, data_dir):
lines = self._read_tsv(os.path.join(data_dir, "online_shopping_test.tsv"))
return self._create_examples(lines, 'test')
@staticmethod
def get_labels():
return ["0", "1"]
# return ['蒙牛', '水果', '洗发水', '平板', '酒店', '手机', '计算机', '书籍', '衣服', '热水器']
@staticmethod
def _create_examples(lines, set_type):
examples = []
for (i, line) in enumerate(lines):
if i == 0:
continue
guid = "%s-%s" % (set_type, i)
text = tokenization.convert_to_unicode(line[3])
label = tokenization.convert_to_unicode(line[2])
examples.append(
InputExample(guid=guid, text=text, label=label))
return examples
@classmethod
def _read_tsv(cls, input_file, quotechar=None):
"""Reads a tab separated value file."""
with tf.gfile.Open(input_file, "r") as f:
reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
lines = []
for line in reader:
lines.append(line)
return lines
得到tfrecord文件
也是模仿Bert的写法写的（甚至保留了打印前五条example）。
# 将一个example类的训练数据转成feature类
def convert_single_example(ex_index, example, tokenizer):
text = example.text
tokens = tokenizer.tokenize(text)
input_ids = tokenizer.convert_tokens_to_ids(tokens)
input_size = len(input_ids)
label = int(example.label)
# 打印前5条转换的记录
if ex_index < 5:
tf.logging.info("*** Example ***")
tf.logging.info("guid: %s" % example.guid)
tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
tf.logging.info("input_size: %s" % input_size)
tf.logging.info("label: %s" % label)
feature = InputFeatures(input_ids=input_ids, input_size=input_size, label=label)
return feature
# 将准备喂入模型的数据存成tfrecord文件
def file_based_convert_examples_to_features(examples, tokenizer, output_file):
writer = tf.python_io.TFRecordWriter(output_file)
for (ex_index, example) in enumerate(examples):
if ex_index % 10000 == 0:
tf.logging.info("Writing example %d of %d" % (ex_index, len(examples)))
feature = convert_single_example(ex_index, example, tokenizer)
def create_int_feature(values):
f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values)))
return f
features = collections.OrderedDict()
features['input_ids'] = create_int_feature(feature.input_ids)
features['input_size'] = create_int_feature([feature.input_size])
features['label'] = create_int_feature([feature.label])
tf_example = tf.train.Example(features=tf.train.Features(feature=features))
writer.write(tf_example.SerializeToString())
writer.close()
读取并解析tfrecord文件
终于又有相对原创的代码了。这里在做batch的同时对input_ids做了padding的处理。效果即input_fn_builder，只是改了名字，调整了结构。
def file_based_dataset_loader(input_file, is_training, batch_size, num_epochs):
# TfRecord的解析函数
def parse_func(serialized_example):
name_to_features = {
"input_ids": tf.VarLenFeature(tf.int64),
"input_size": tf.FixedLenFeature(shape=(1,), dtype=tf.int64),
"label": tf.FixedLenFeature(shape=(1,), dtype=tf.int64),
}
parsed_example = tf.parse_single_example(serialized_example, features=name_to_features)
parsed_example['input_ids'] = tf.sparse_tensor_to_dense(parsed_example['input_ids'])
input_ids = parsed_example['input_ids']
input_size = parsed_example['input_size']
label = parsed_example['label']
return input_ids, input_size, label
dataset = tf.data.TFRecordDataset(input_file)
dataset = dataset.map(parse_func)
if is_training:
dataset = dataset.repeat(num_epochs).shuffle(buffer_size=100)
padded_shapes = (tf.TensorShape([None]), # 语料数据，None即代表batch_size
tf.TensorShape([None]), # 语料数据各个句子的原始长度
tf.TensorShape([None])) # 标签数据，None即代表batch_size
# 调用padded_batch方法进行batching操作
batched_dataset = dataset.padded_batch(batch_size, padded_shapes)
# dataset = dataset.batch(batch_size)
return batched_dataset
正式运行
模型运行的函数，单独拿出来。
def run_epoch(session, cost_op, accuracy_op, train_op, step):
while True:
try:
cost, accuracy, _ = session.run([cost_op, accuracy_op, train_op])
if step % 100 == 0:
tf.logging.info('Steps: {0}, Loss value: {1}，Accuracy: {2}'.format(
step, cost, accuracy))
step += 1
except tf.errors.OutOfRangeError:
break
主程序。在config配置里直接写明了词典文件的词数。
另外，不管是在构建词典文件，处理待预测的句子，都没有做去停用词的步骤。
def main():
config = modelling.LstmConfig(vocab_size=68355)
tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file)
processor = DataProcessor()
if FLAGS.do_train:
# 定义训练用循环神经网络模型
train_model = modelling.LstmModel(config, mode='train')
train_examples = processor.get_train_examples(FLAGS.data_dir)
num_train_steps = int(len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs)
train_file = r'./tmp/train.tfrecord'
file_based_convert_examples_to_features(train_examples, tokenizer, train_file)
# 获得训练数据
train_data = file_based_dataset_loader(train_file,
is_training=True,
batch_size=FLAGS.train_batch_size,
num_epochs=FLAGS.num_train_epochs)
train_iterator = train_data.make_initializable_iterator()
input_ids, input_sizes, labels = train_iterator.get_next()
input_sizes = tf.reshape(input_sizes, shape=(-1,))
labels = tf.reshape(labels, shape=(-1,))
# 定义前向计算图，输入数据以张量的形式提供给forward函数
_, cost_op, accuracy_op, train_op = train_model.forward(input_ids, input_sizes, labels)
#
# TensorFlow持久化类
saver = tf.train.Saver()
step = 0
# 训练模型
tf.logging.info("***** Running training *****")
tf.logging.info(" Num examples = %d", len(train_examples))
tf.logging.info(" Total training steps: {}".format(num_train_steps))
with tf.Session() as sess:
tf.global_variables_initializer().run()
sess.run(train_iterator.initializer)
run_epoch(sess, cost_op, accuracy_op, train_op, step)
tf.logging.info("********* Training Step Finished *********")
saver.save(sess, r'./model/lstm.ckpt')
if FLAGS.do_eval:
tf.reset_default_graph()
eval_model = modelling.LstmModel(config, mode='eval')
eval_examples = processor.get_dev_examples(FLAGS.data_dir)
eval_file = r'./tmp/eval.tfrecord'
file_based_convert_examples_to_features(eval_examples, tokenizer, eval_file)
# 获得测试数据
eval_data = file_based_dataset_loader(eval_file,
is_training=False,
batch_size=FLAGS.eval_batch_size,
num_epochs=None)
eval_iterator = eval_data.make_initializable_iterator()
eval_input_ids, eval_input_sizes, eval_labels = eval_iterator.get_next()
eval_input_sizes = tf.reshape(eval_input_sizes, shape=(-1,))
eval_labels = tf.reshape(eval_labels, shape=(-1,))
eval_cost_op, eval_accuracy_op = eval_model.forward(eval_input_ids, eval_input_sizes, eval_labels)
# 测试模型
eval_step = 0
tf.logging.info("***** Running Eval *****")
tf.logging.info(" Num examples = %d", len(eval_examples))
with tf.Session() as sess_eval:
tf.global_variables_initializer().run()
sess_eval.run(eval_iterator.initializer)
saver = tf.train.Saver()
saver.restore(sess_eval, r'./model/lstm.ckpt')
tf.logging.info("*** Restore finished ***")
eval_cost = 0
eval_correct = 0
while True:
try:
cost, accuracy = sess_eval.run([eval_cost_op, eval_accuracy_op])
eval_cost += cost / FLAGS.eval_batch_size
eval_correct += accuracy * FLAGS.eval_batch_size
eval_step += 1
except tf.errors.OutOfRangeError:
break
eval_cost = eval_cost / (len(eval_examples) / FLAGS.eval_batch_size)
eval_accuracy = eval_correct / len(eval_examples)
print('Eval Cost: {0}, Eval Accuracy: {1}'.format(eval_cost, eval_accuracy))
with open(r'./model/eval_result.txt', 'w', encoding='utf-8') as f:
f.write('Eval Cost: {0}, Eval Accuracy: {1}'.format(eval_cost, eval_accuracy))
if __name__ == '__main__':
main()
调用模型完成预测
重新写了convert_single_example方法。
由于只是少量单句的预测，因此没有做生成tfrecord并解析的处理，而是直接做了padding的处理。
import tensorflow as tf
import tokenization
import modelling
from run_classifier import InputExample, InputFeatures
# 将一个example类的训练数据转成feature类
def convert_single_example(example, tokenizer):
text = example.text
tokens = tokenizer.tokenize(text)
input_ids = tokenizer.convert_tokens_to_ids(tokens)
input_size = len(input_ids)
feature = InputFeatures(input_ids=input_ids, input_size=input_size, label=None)
return feature
def main(data):
tokenizer = tokenization.FullTokenizer(vocab_file=r'./vocab.txt')
config = modelling.LstmConfig(vocab_size=68355)
input_ids = []
input_sizes = []
max_length = 0
for index in range(len(data)):
guid = 'test-%d' % index
text = tokenization.convert_to_unicode(str(data[index]))
data_example = InputExample(guid=guid, text=text, label=None)
data_feature = convert_single_example(example=data_example, tokenizer=tokenizer)
print(data_feature.input_ids)
if len(data_feature.input_ids) > max_length:
max_length = len(data_feature.input_ids)
input_ids.append(data_feature.input_ids)
input_sizes.append(data_feature.input_size)
for input_id in input_ids:
if len(input_id) < max_length:
input_id.extend((max_length-len(input_id)) * [0])
input_ids = tf.convert_to_tensor(input_ids)
input_sizes = tf.convert_to_tensor(input_sizes)
predict_model = modelling.LstmModel(config, mode='predict')
output_op = predict_model.forward(input_ids, input_sizes, label=None)
with tf.Session() as sess:
saver = tf.train.Saver()
saver.restore(sess, r'./model/lstm.ckpt')
output = sess.run(output_op)
print(output)
print(tf.argmax(output, 1).eval())
if __name__ == '__main__':
text = ['这个房间真的很棒，又舒服又便宜', '这个房间太差了，又贵又破，不推荐', '一句很普通很中立的话']
main(text)
结果
同样的训练集与测试集，在Bert上跑完的准确度是0.946，用lstm，在2个epoch上且没有调整学习率的情况下，跑完准确度有0.897。其中Bert模型在服务器上跑了三天两夜，lstm在本机cpu上跑了一个半小时。结果还可以。
To Do
tf.estimator，这个高级封装的api似乎很牛逼，Bert中也使用了这个api，要继续学习学习。牵扯到model_fn的构建。我理解应该是重新复构模型的结构。在modelling中应该只定义模型结构，对于前向传播的过程，都到model_fn中完成。但是具体的操作还需要进一步的了解。
不得不说，虽然这样的代码可能还是不算高级，但是已经比我之前自己写的初版，在结构上要清楚太多了。同时也对tf.dataset有了比较深的了解（之前的next_batch都是完全自己写的，这里用dataset的iterator就可以搞定）。やはり多读读牛逼的源码是很有用的。
