# embedding层单词和分词信息
embedding = self.embedding_layer(self.word_inputs, self.seg_inputs, config)
# lstm输入层
lstm_inputs = tf.nn.dropout(embedding, self.dropout)
# lstm输出层
lstm_outputs = self.biLSTM_layer(lstm_inputs, self.lstm_dim, self.lengths)
# 投射层
self.logits = self.project_layer(lstm_outputs)
# 损失
self.loss = self.crf_loss_layer(self.logits, self.lengths)
tf.nn.embedding_lookup的作用就是找到要寻找的embedding data中的对应的行下的vector。
image.png
def embedding_layer(self, word_inputs, seg_inputs, config, name=None):
"""
:param word_inputs: one-hot编码.其实所有字的one_hot编码
:param seg_inputs: 分词特征
:param config: 配置
:param name: 层的命名
:return: shape = [word_inputs,word_dim+seg_dim]
"""
embedding = []
with tf.variable_scope("word_embedding" if not name else name), tf.device('/cpu:0'):
self.word_lookup = tf.get_variable(
name="word_embedding",
shape=[self.num_words, self.word_dim],
initializer=self.initializer
)
embedding.append(tf.nn.embedding_lookup(self.word_lookup, word_inputs))
if config['seg_dim']:
with tf.variable_scope("seg_embedding"), tf.device('/cpu:0'):
self.seg_lookup = tf.get_variable(
name="seg_embedding",
shape=[self.num_sges, self.seg_dim],
initializer=self.initializer
)
embedding.append(tf.nn.embedding_lookup(self.seg_lookup, seg_inputs))
embed = tf.concat(embedding, axis=-1)
return embed
image.png
def biLSTM_layer(self, lstm_inputs, lstm_dim, lengths, name=None):
"""
:param lstm_inputs: [batch_size, num_steps, emb_size]
:param lstm_dim:
:param name:
:return: [batch_size, num_steps, 2*lstm_dim]
为何返回是2*lstm_dim，因为其是双向的lstm。每个方向的输出为lstm_dim
"""
with tf.variable_scope("word_biLSTM" if not name else name):
lstm_cell = {}
for direction in ['forward', 'backward']:
with tf.variable_scope(direction):
lstm_cell[direction] = rnn.CoupledInputForgetGateLSTMCell(
lstm_dim,
use_peepholes=True,
initializer=self.initializer,
state_is_tuple=True
)
outputs, final_status = tf.nn.bidirectional_dynamic_rnn(
lstm_cell['forward'],
lstm_cell['backward'],
lstm_inputs,
dtype=tf.float32,
sequence_length=lengths
)
# 因为单向的lstm输出的格式为[batch_size, num_steps,lstm_dim]。
# 2表示在lstm_dim这个维度进行拼接。
# 个人觉得outputs的输出格式为[[batch_size, num_steps,lstm_dim],[batch_size, num_steps,lstm_dim]]
# 即是一个list。list里面的每一个元素是单向的lstm的输出
return tf.concat(outputs, axis=2)
def project_layer(self, lstm_outputs, name=None):
"""
:param lstm_outputs: [batch_size, num_steps, emb_size]
个人觉得lstm_outputs: [batch_size, num_steps, lstm_dim * 2] num_steps表示每个句子里面字的数量。即每个批次的句子长度
:param name:
:return: [batch_size,num_steps, num_tags]
"""
with tf.variable_scope('project_layer' if not name else name):
with tf.variable_scope('hidden_layer'):
W = tf.get_variable(
"W",
shape=[self.lstm_dim * 2, self.lstm_dim],
dtype=tf.float32,
initializer=self.initializer
)
b = tf.get_variable(
"b",
shape=[self.lstm_dim],
dtype=tf.float32,
initializer=tf.zeros_initializer()
)
out_put = tf.reshape(lstm_outputs, shape=[-1, self.lstm_dim * 2]) # 得到所有的字，将所有的字最后编码为lstm_dim长度
hidden = tf.tanh(tf.nn.xw_plus_b(out_put, W, b))
with tf.variable_scope('logits'):
W = tf.get_variable(
"W",
shape=[self.lstm_dim, self.num_tags],
dtype=tf.float32,
initializer=self.initializer
)
b = tf.get_variable(
"b",
shape=[self.num_tags],
dtype=tf.float32,
initializer=tf.zeros_initializer()
)
# 最后将每个字编码为num_tags。即最后想要得到每个字属于每个tag的概率
pred = tf.nn.xw_plus_b(hidden, W, b)
# 返回原始的shape。即batch_size,num_setps,num_tags
return tf.reshape(pred, [-1, self.num_setps, self.num_tags])
+CRF
def crf_loss_layer(self, project_logits, lenghts, name=None):
"""
# 个人觉得是[-1, self.num_setps, self.num_tags]
:param project_logits: [1, num_steps, num_tages]
:param lenghts:
:param name:
:return: scalar loss
听说下面是固定的写法
"""
with tf.variable_scope('crf_loss' if not name else name):
small_value = -10000.0
# 下面是对于一个字。但是最后一维，比原来的标签长度多了一个元素
start_logits = tf.concat(
[
small_value * tf.ones(shape=[self.batch_size, 1, self.num_tags]),
tf.zeros(shape=[self.batch_size, 1, 1])
],
axis=-1
)
pad_logits = tf.cast(
small_value *
tf.ones(shape=[self.batch_size, self.num_setps, 1]),
dtype=tf.float32
)
# 貌似是在列的位置最后拼接一个元素.所以此时project_layer层输出的每个字最后一层多了一个元素
# 即在最后一个维度填充了一个元素
logits = tf.concat(
[project_logits, pad_logits],
axis=-1
)
# 此时相当于在每个批次的，每个句子开头位置添加了一个字
logits = tf.concat(
[start_logits, logits],
axis=1
)
# 因为self.targets.shape = [batch_size,num_steps].所以下面的操作，类似于在每个句子前面添加了一个字
# 所以此时就和上面的填充的形状tf.concat([start_logits, logits],axis=1)
# 对应了起来
targets = tf.concat(
[tf.cast(
self.num_tags * tf.ones([self.batch_size, 1]),
tf.int32
),
self.targets
]
,
axis=-1
)
# 每个状态之间的转移矩阵
self.trans = tf.get_variable(
"transitions",
shape=[self.num_tags + 1, self.num_tags + 1],
initializer=self.initializer
)
log_likehood, self.trans = crf_log_likelihood(
inputs=logits,
tag_indices=targets,
transition_params=self.trans,
sequence_lengths=lenghts + 1 # 因为上面在句子的开头位置添加了一个字
)
return tf.reduce_mean(-log_likehood)
用F1值来评估
def evaluate(sess, model, name, manager, id_to_tag, logger):
logger.info('evaluate:{}'.format(name))
ner_results = model.evaluate(sess, manager, id_to_tag)
eval_lines = model_utils.test_ner(ner_results, FLAGS.result_path)
for line in eval_lines:
logger.info(line)
f1 = float(eval_lines[1].strip().split()[-1])
if name == "dev":
best_test_f1 = model.best_dev_f1.eval()
if f1 > best_test_f1:
tf.assign(model.best_dev_f1, f1).eval()
logger.info('new best dev f1 socre:{:>.3f}'.format(f1))
return f1 > best_test_f1
elif name == "test":
best_test_f1 = model.best_test_f1.eval()
if f1 > best_test_f1:
tf.assign(model.best_test_f1, f1).eval()
logger.info('new best test f1 score:{:>.3f}'.format(f1))
return f1 > best_test_f1
关于调参：
Validation loss vs Training Loss
如果validation loss < Training Loss, 可能就过拟合了。这样就需要尝试着降低网络大小network size 或者 提高dropout的值，比如0.5,0.6依次尝试。用mini batch的方法，把数据集划分成很若干个小一点的集合。来调整参数：如embedding_dim, lstm_dim, learning=rate(3e-4)
这里用CRF++先跑了一遍，速度很快，准确率在0.8左右，recall在0.87左右，f1在0.87多。然后用BiLSTM后接softmax来跑loss一下子降到很低，感觉很容易局部过拟合。BiLSTM+CRF后， loss稳定变小，到0.15时候准确率变化已经比较少了，比不接CRF的更快拟合。总体准确率比无CRF的更高。另外，迭代次数调高后，准确率也会提高一点。
