卷积姿态机.
卷积姿态机 (Convolutional Pose Machine, CPM)是一个为人体姿态估计任务设计的卷积神经网络模型。它使用卷积网络来进行特征提取和上下文信息提，以置信度热图的形式表示预测结果（能够保留空间信息），在全卷积的结构下使用中间监督进行端到端的训练和测试，极大提高了关键点检测的准确率。
CPM使用顺序化的卷积架构来表达空间信息和纹理信息。顺序化的卷积架构表现在网络分为多个阶段，每一个阶段都有监督训练的部分。前面的阶段使用原始图片作为输入，后面阶段使用之前阶段的特征图作为输入，主要是为了融合空间信息、纹理信息和中心约束。另外，对同一个卷积架构同时使用多个尺度处理输入的特征和响应，既能保证精度，又考虑了各部件之间的远近距离关系。
CPM采用的顺序化网络结构如下：
- 阶段1：输入是原始图像，经过预训练模型（如VGGNet）提取图像特征$x$，再经过两个$1\times 1$卷积输出尺寸为$h’\times w’\times (P+1)$的关节点热图，$P+1$通道表示热图上每个像素位置是$P$个关键点$+1$个背景的得分。
- 阶段2：输入包含三部分：原图、阶段1输出的热图、每个目标的中心约束图； 输出的是$P+1$通道的热图。
- 阶段3及以后：与阶段2相似，但不再输入原图进行图像特征的计算，而是直接对阶段2的中间卷积特征进行卷积来计算图像特征。
其中阶段2之后输入的中心约束图是使用图像中各个目标中心位置处高斯撒点生成的，用于约束各个目标的位置。
class CPM(nn.Module):
def __init__(self, k):
super(CPM, self).__init__()
self.k = k
self.pool_center = nn.AvgPool2d(kernel_size=9, stride=8, padding=1)
self.conv1_stage1 = nn.Conv2d(3, 128, kernel_size=9, padding=4)
self.pool1_stage1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.conv2_stage1 = nn.Conv2d(128, 128, kernel_size=9, padding=4)
self.pool2_stage1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.conv3_stage1 = nn.Conv2d(128, 128, kernel_size=9, padding=4)
self.pool3_stage1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.conv4_stage1 = nn.Conv2d(128, 32, kernel_size=5, padding=2)
self.conv5_stage1 = nn.Conv2d(32, 512, kernel_size=9, padding=4)
self.conv6_stage1 = nn.Conv2d(512, 512, kernel_size=1)
self.conv7_stage1 = nn.Conv2d(512, self.k + 1, kernel_size=1)
self.conv1_stage2 = nn.Conv2d(3, 128, kernel_size=9, padding=4)
self.pool1_stage2 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.conv2_stage2 = nn.Conv2d(128, 128, kernel_size=9, padding=4)
self.pool2_stage2 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.conv3_stage2 = nn.Conv2d(128, 128, kernel_size=9, padding=4)
self.pool3_stage2 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.conv4_stage2 = nn.Conv2d(128, 32, kernel_size=5, padding=2)
self.Mconv1_stage2 = nn.Conv2d(32 + self.k + 2, 128, kernel_size=11, padding=5)
self.Mconv2_stage2 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
self.Mconv3_stage2 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
self.Mconv4_stage2 = nn.Conv2d(128, 128, kernel_size=1, padding=0)
self.Mconv5_stage2 = nn.Conv2d(128, self.k + 1, kernel_size=1, padding=0)
self.conv1_stage3 = nn.Conv2d(128, 32, kernel_size=5, padding=2)
self.Mconv1_stage3 = nn.Conv2d(32 + self.k + 2, 128, kernel_size=11, padding=5)
self.Mconv2_stage3 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
self.Mconv3_stage3 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
self.Mconv4_stage3 = nn.Conv2d(128, 128, kernel_size=1, padding=0)
self.Mconv5_stage3 = nn.Conv2d(128, self.k + 1, kernel_size=1, padding=0)
self.conv1_stage4 = nn.Conv2d(128, 32, kernel_size=5, padding=2)
self.Mconv1_stage4 = nn.Conv2d(32 + self.k + 2, 128, kernel_size=11, padding=5)
self.Mconv2_stage4 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
self.Mconv3_stage4 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
self.Mconv4_stage4 = nn.Conv2d(128, 128, kernel_size=1, padding=0)
self.Mconv5_stage4 = nn.Conv2d(128, self.k + 1, kernel_size=1, padding=0)
self.conv1_stage5 = nn.Conv2d(128, 32, kernel_size=5, padding=2)
self.Mconv1_stage5 = nn.Conv2d(32 + self.k + 2, 128, kernel_size=11, padding=5)
self.Mconv2_stage5 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
self.Mconv3_stage5 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
self.Mconv4_stage5 = nn.Conv2d(128, 128, kernel_size=1, padding=0)
self.Mconv5_stage5 = nn.Conv2d(128, self.k + 1, kernel_size=1, padding=0)
self.conv1_stage6 = nn.Conv2d(128, 32, kernel_size=5, padding=2)
self.Mconv1_stage6 = nn.Conv2d(32 + self.k + 2, 128, kernel_size=11, padding=5)
self.Mconv2_stage6 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
self.Mconv3_stage6 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
self.Mconv4_stage6 = nn.Conv2d(128, 128, kernel_size=1, padding=0)
self.Mconv5_stage6 = nn.Conv2d(128, self.k + 1, kernel_size=1, padding=0)
def _stage1(self, image):
x = self.pool1_stage1(F.relu(self.conv1_stage1(image)))
x = self.pool2_stage1(F.relu(self.conv2_stage1(x)))
x = self.pool3_stage1(F.relu(self.conv3_stage1(x)))
x = F.relu(self.conv4_stage1(x))
x = F.relu(self.conv5_stage1(x))
x = F.relu(self.conv6_stage1(x))
x = self.conv7_stage1(x)
return x
def _middle(self, image):
x = self.pool1_stage2(F.relu(self.conv1_stage2(image)))
x = self.pool2_stage2(F.relu(self.conv2_stage2(x)))
x = self.pool3_stage2(F.relu(self.conv3_stage2(x)))
return x
def _stage2(self, pool3_stage2_map, conv7_stage1_map, pool_center_map):
x = F.relu(self.conv4_stage2(pool3_stage2_map))
x = torch.cat([x, conv7_stage1_map, pool_center_map], dim=1)
x = F.relu(self.Mconv1_stage2(x))
x = F.relu(self.Mconv2_stage2(x))
x = F.relu(self.Mconv3_stage2(x))
x = F.relu(self.Mconv4_stage2(x))
x = self.Mconv5_stage2(x)
return x
def _stage3(self, pool3_stage2_map, Mconv5_stage2_map, pool_center_map):
x = F.relu(self.conv1_stage3(pool3_stage2_map))
x = torch.cat([x, Mconv5_stage2_map, pool_center_map], dim=1)
x = F.relu(self.Mconv1_stage3(x))
x = F.relu(self.Mconv2_stage3(x))
x = F.relu(self.Mconv3_stage3(x))
x = F.relu(self.Mconv4_stage3(x))
x = self.Mconv5_stage3(x)
return x
def _stage4(self, pool3_stage2_map, Mconv5_stage3_map, pool_center_map):
x = F.relu(self.conv1_stage4(pool3_stage2_map))
x = torch.cat([x, Mconv5_stage3_map, pool_center_map], dim=1)
x = F.relu(self.Mconv1_stage4(x))
x = F.relu(self.Mconv2_stage4(x))
x = F.relu(self.Mconv3_stage4(x))
x = F.relu(self.Mconv4_stage4(x))
x = self.Mconv5_stage4(x)
return x
def _stage5(self, pool3_stage2_map, Mconv5_stage4_map, pool_center_map):
x = F.relu(self.conv1_stage5(pool3_stage2_map))
x = torch.cat([x, Mconv5_stage4_map, pool_center_map], dim=1)
x = F.relu(self.Mconv1_stage5(x))
x = F.relu(self.Mconv2_stage5(x))
x = F.relu(self.Mconv3_stage5(x))
x = F.relu(self.Mconv4_stage5(x))
x = self.Mconv5_stage5(x)
return x
def _stage6(self, pool3_stage2_map, Mconv5_stage5_map, pool_center_map):
x = F.relu(self.conv1_stage6(pool3_stage2_map))
x = torch.cat([x, Mconv5_stage5_map, pool_center_map], dim=1)
x = F.relu(self.Mconv1_stage6(x))
x = F.relu(self.Mconv2_stage6(x))
x = F.relu(self.Mconv3_stage6(x))
x = F.relu(self.Mconv4_stage6(x))
x = self.Mconv5_stage6(x)
return x
def forward(self, image, center_map):
pool_center_map = self.pool_center(center_map)
conv7_stage1_map = self._stage1(image)
pool3_stage2_map = self._middle(image)
Mconv5_stage2_map = self._stage2(pool3_stage2_map, conv7_stage1_map,
pool_center_map)
Mconv5_stage3_map = self._stage3(pool3_stage2_map, Mconv5_stage2_map,
pool_center_map)
Mconv5_stage4_map = self._stage4(pool3_stage2_map, Mconv5_stage3_map,
pool_center_map)
Mconv5_stage5_map = self._stage5(pool3_stage2_map, Mconv5_stage4_map,
pool_center_map)
Mconv5_stage6_map = self._stage6(pool3_stage2_map, Mconv5_stage5_map,
pool_center_map)
return conv7_stage1_map, Mconv5_stage2_map, Mconv5_stage3_map, Mconv5_stage4_map, Mconv5_stage5_map, Mconv5_stage6_map
如果直接在最后使用损失函数对整个网络进行优化，梯度在反向传播时很容易发生梯度消失，在浅层梯度会趋近于0。CPM采用中间监督的方法来解决，每一阶段的输出热图都对应有标签来计算损失函数，完成中间监督。
中间监督的优势在于即使整个网络有很多层，也不会出现梯度消失，因为中间监督会在每个阶段补充梯度。下图给出了是/否带有中间监督的情况下，训练过程中不同网络深度处的梯度强度直方图。
- 在训练初期，最后一层的梯度分布方差比较大。如果没有中间监督，浅层的梯度都集中在0附近（梯度消失）。如果有中间监督，浅层的梯度分布方差大，表明中间监督确实对梯度消失问题有帮助。
- 在训练后期，没有中间监督时层数越浅，梯度消失仍然越严重；而有中间监督的情况下，浅层的梯度分布方差没有训练初期那么大，但是也没消失，这表明了模型的收敛。
