总结
DataLayerSetUp
首先，cpm_data_layer.cpp调用DataLayerSetUp函数设置层参数
template <typename Dtype>
void CPMDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top)
transformed_label_在这里被Reshape为以下维度，其中num_parts为56
this->transformed_label_.Reshape(1, 2*(num_parts+1), height/stride, width/stride); //Line:91
load_batch
接下来调用load_batch函数，
template<typename Dtype>
void CPMDataLayer<Dtype>::load_batch(Batch<Dtype>* batch)
Transform函数未调用
因为 datum.encoded() = false（encoded详见训练过程（4））
if (datum.encoded()) {
this->cpm_data_transformer_->Transform(cv_img, &(this->transformed_data_));
}
Transform_nv
调用Transform_nv函数
else {
this->cpm_data_transformer_->Transform_nv(datum, //Datum& datum: 应该为读入的数据
&(this->transformed_data_), //this->transformed_data_.set_cpu_data(top_data + offset_data);
&(this->transformed_label_), cnt); //this->transformed_label_.set_cpu_data(top_label + offset_label);
++cnt; //cnt在batch_size for循环外初始值为0，for循环内自增1
}
template<typename Dtype> void CPMDataTransformer<Dtype>::Transform_nv(const Datum& datum, Dtype* transformed_data, Dtype* transformed_label, int cnt)
ReadMetaData(meta, data, offset3, offset1); //data = datum.data() Line:514
ReadMetaData(meta, data, offset3, offset1); 将data中的数据按顺序读入到meta中，类似于genLMDB.py生成的output.txt数据格式
TransformMetaJoints
TransformMetaJoints(meta);
17个关节点变为18个
JSON_17num_parts.png
void CPMDataTransformer<Dtype>::TransformJoints(Joints& j) {
//transform joints in meta from np_in_lmdb (specified in prototxt) to np (specified in prototxt)
Joints jo = j;
if(np == 56){
int COCO_to_ours_1[18] = {1,6, 7,9,11, 6,8,10, 13,15,17, 12,14,16, 3,2,5,4}; //17个关节点变为18个
int COCO_to_ours_2[18] = {1,7, 7,9,11, 6,8,10, 13,15,17, 12,14,16, 3,2,5,4};
jo.joints.resize(np);
jo.isVisible.resize(np);
for(int i=0;i<18;i++){
jo.joints[i] = (j.joints[COCO_to_ours_1[i]-1] + j.joints[COCO_to_ours_2[i]-1]) * 0.5;
if(j.isVisible[COCO_to_ours_1[i]-1]==2 || j.isVisible[COCO_to_ours_2[i]-1]==2){
jo.isVisible[i] = 2;
}
else if(j.isVisible[COCO_to_ours_1[i]-1]==3 || j.isVisible[COCO_to_ours_2[i]-1]==3){
jo.isVisible[i] = 3;
}
else {
jo.isVisible[i] = j.isVisible[COCO_to_ours_1[i]-1] && j.isVisible[COCO_to_ours_2[i]-1];
}
}
}
generateLabelMap
generateLabelMap(transformed_label, img_aug, meta);
void CPMDataTransformer<Dtype>::generateLabelMap(Dtype* transformed_label, Mat& img_aug, MetaData meta)
放置高斯响应，放置高斯响应函数比较简单（至于transformed_label为什么要从[(np+1) * channelOffset + g_y * grid_x + g_x]开始，是因为在generateLabelMap函数之前，被mask_miss和一个background占了
if (mode > 4){
for (int g_y = 0; g_y < grid_y; g_y++){
for (int g_x = 0; g_x < grid_x; g_x++){
for (int i = 0; i < np; i++){
float weight = float(mask_miss_aug.at<uchar>(g_y, g_x)) /255; //mask_miss_aug.at<uchar>(i, j);
if (meta.joint_self.isVisible[i] != 3){
transformed_labeld[i*channelOffset + g_y*grid_x + g_x] = weight;
}
}
// background channel
if(mode == 5){
transformed_label[np*channelOffset + g_y*grid_x + g_x] = float(mask_miss_aug.at<uchar>(g_y, g_x)) /255;
}
if(mode > 5){
transformed_label[np*channelOffset + g_y*grid_x + g_x] = 1;
transformed_label[(2*np+1)*channelOffset + g_y*grid_x + g_x] = float(mask_all_aug.at<uchar>(g_y, g_x)) /255;
}
}
}
}
for (int g_y = 0; g_y < grid_y; g_y++){
for (int g_x = 0; g_x < grid_x; g_x++){
for (int i = np+1; i < 2*(np+1); i++){
if (mode == 6 && i == (2*np + 1))
continue;
transformed_label[i*channelOffset + g_y*grid_x + g_x] = 0;
}
}
}
if (np == 56){
for (int i = 0; i < 18; i++){
Point2f center = meta.joint_self.joints[i];
if(meta.joint_self.isVisible[i] <= 1){
putGaussianMaps(transformed_label + (i+np+39)*channelOffset, center, param_.stride(),
grid_x, grid_y, param_.sigma()); //self 放置关节点高斯响应
}
for(int j = 0; j < meta.numOtherPeople; j++){ //for every other person
Point2f center = meta.joint_others[j].joints[i];
if(meta.joint_others[j].isVisible[i] <= 1){
putGaussianMaps(transformed_label + (i+np+39)*channelOffset, center, param_.stride(),
grid_x, grid_y, param_.sigma());
}
}
}
2×19（PAF）的数组排序
int mid_1[19] = {2, 9, 10, 2, 12, 13, 2, 3, 4, 3, 2, 6, 7, 6, 2, 1, 1, 15, 16};
int mid_2[19] = {9, 10, 11, 12, 13, 14, 3, 4, 5, 17, 6, 7, 8, 18, 1, 15, 16, 17, 18};
int thre = 1;
for(int i=0;i<19;i++){ // 2×19=38 2×PAF
Mat count = Mat::zeros(grid_y, grid_x, CV_8UC1);
Joints jo = meta.joint_self;
if(jo.isVisible[mid_1[i]-1]<=1 && jo.isVisible[mid_2[i]-1]<=1){
//putVecPeaks
putVecMaps(transformed_label + (np+ 1+ 2*i)*channelOffset, transformed_label + (np+ 2+ 2*i)*channelOffset,
count, jo.joints[mid_1[i]-1], jo.joints[mid_2[i]-1], param_.stride(), grid_x, grid_y, param_.sigma(), thre); //self
} //与COCO对应
for(int j = 0; j < meta.numOtherPeople; j++){ //for every other person
Joints jo2 = meta.joint_others[j];
if(jo2.isVisible[mid_1[i]-1]<=1 && jo2.isVisible[mid_2[i]-1]<=1){
//putVecPeaks
putVecMaps(transformed_label + (np+ 1+ 2*i)*channelOffset, transformed_label + (np+ 2+ 2*i)*channelOffset,
count, jo2.joints[mid_1[i]-1], jo2.joints[mid_2[i]-1], param_.stride(), grid_x, grid_y, param_.sigma(), thre); //self
}
}
}
putVecMaps函数用于设置PAF的labels，count初始值为0
Mat count = Mat::zeros(grid_y, grid_x, CV_8UC1);
void CPMDataTransformer<Dtype>::putVecMaps(Dtype* entryX, Dtype* entryY, Mat& count, Point2f centerA, Point2f centerB, int stride, int grid_x, int grid_y, float sigma, int thre)
