写入Tfrecord
print("convert data into tfrecord:train\n") out_file_train = "/home/huadong.wang/bo.yan/fudan_mtl/data/ace2005/bn_nw.train.tfrecord" writer = tf.python_io.TFRecordWriter(out_file_train) for i in tqdm(range(len(data_train))): record = tf.train.Example(features=tf.train.Features(feature={ 'word_ids': tf.train.Feature(bytes_list=tf.train.BytesList(value=[train_x[i].tostring()])), 'et_ids1': tf.train.Feature(bytes_list=tf.train.BytesList(value=[train_et1[i].tostring()])), 'et_ids2': tf.train.Feature(bytes_list=tf.train.BytesList(value=[train_et2[i].tostring()])), 'position_ids1': tf.train.Feature(bytes_list=tf.train.BytesList(value=[train_p1[i].tostring()])), 'position_ids2': tf.train.Feature(bytes_list=tf.train.BytesList(value=[train_p1[i].tostring()])), 'chunks': tf.train.Feature(bytes_list=tf.train.BytesList(value=[train_chunks[i].tostring()])), 'spath_ids': tf.train.Feature(bytes_list=tf.train.BytesList(value=[train_spath[i].tostring()])), 'seq_len': tf.train.Feature(int64_list=tf.train.Int64List(value=[train_x_len[i]])), 'label': tf.train.Feature(int64_list=tf.train.Int64List(value=[np.argmax(train_relation[i])])), 'task': tf.train.Feature(int64_list=tf.train.Int64List(value=[np.int64(0)])) })) writer.write(record.SerializeToString()) writer.close()
解析tfrecord
def _parse_tfexample(serialized_example): '''parse serialized tf.train.SequenceExample to tensors context features : label, task sequence features: sentence ''' context_features={'label' : tf.FixedLenFeature([], tf.int64), 'task' : tf.FixedLenFeature([], tf.int64), 'seq_len': tf.FixedLenFeature([], tf.int64)} sequence_features={'word_ids': tf.FixedLenSequenceFeature([], tf.int64), 'et_ids1': tf.FixedLenSequenceFeature([], tf.int64), 'et_ids2': tf.FixedLenSequenceFeature([], tf.int64), 'position_ids1': tf.FixedLenSequenceFeature([], tf.int64), 'position_ids2': tf.FixedLenSequenceFeature([], tf.int64), 'chunks': tf.FixedLenSequenceFeature([], tf.int64), 'spath_ids': tf.FixedLenSequenceFeature([], tf.int64), } context_dict, sequence_dict = tf.parse_single_sequence_example( serialized_example, context_features = context_features, sequence_features = sequence_features) sentence = (sequence_dict['word_ids'],sequence_dict['et_ids1'],sequence_dict['et_ids2'],sequence_dict['position_ids1'], sequence_dict['position_ids2'],sequence_dict['chunks'],sequence_dict['spath_ids'], context_dict['seq_len']) label = context_dict['label'] task = context_dict['task'] return task, label, sentencedef read_tfrecord(epoch, batch_size): for dataset in DATASETS: train_record_file = os.path.join(OUT_DIR, dataset+'.train.tfrecord') test_record_file = os.path.join(OUT_DIR, dataset+'.test.tfrecord') train_data = util.read_tfrecord(train_record_file, epoch, batch_size, _parse_tfexample, shuffle=True) test_data = util.read_tfrecord(test_record_file, epoch, batch_size, _parse_tfexample, shuffle=False) yield train_data, test_data
模型中使用：
def build_task_graph(self, data): task_label, labels, sentence = data # sentence = tf.nn.embedding_lookup(self.word_embed, sentence)########################## word_ids, et_ids1,et_ids2,position_ids1,position_ids2,chunks,spath_ids,seq_len = sentence # sentence = word_ids######################### self.word_ids = word_ids self.position_ids1 = position_ids1 self.position_ids2 = position_ids2 self.et_ids1 = et_ids1 self.et_ids2 = et_ids2 self.chunks_ids = chunks self.spath_ids = spath_ids self.seq_len = seq_len sentence = self.add_embedding_layers()