无法理解修改后的 VGG16 前向方法（Pytorch）的行为
Trouble understanding behaviour of modified VGG16 forward method (Pytorch)
我修改了 pytorch 中的 VGG16，以在特征提取器中插入 BN 和 dropout 等内容。当我更改 forward 方法的定义时，我偶然发现了一些奇怪的东西：
def forward(self, x):
x = self.model(x)
return x
至：
def forward(self, x):
x = self.model.features(x)
x = self.model.avgpool(x)
x = self.model.classifier(x)
return x
在第二种方法中，我现在收到矩阵大小不匹配的错误
(mat1 dim 1 must match mat2 dim 0)
下面是我一直在使用的 VGG 编辑版本的完整代码。
class Vgg(nn.Module):
def __init__(self, n_classes, bias= None, dropout = 0.3):
super().__init__()
self.model = models.vgg16()
#self.bn64 = nn.BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
#self.bn128 = nn.BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
#self.bn256 = nn.BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
#self.bn512 = nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
# change to allow 4 channels input
self.model.features[0] = nn.Conv2d(4, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3))
# remove/edit some of the first layers to make it more similar to Resnet
del self.model.features[2]
del self.model.features[2]
del self.model.features[-1]
self.model.features[2] = nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
# add dropout
for m in self.model.modules():
if isinstance(m, nn.Dropout):
m.p = dropout
else:
pass
self.dropout = nn.Dropout(p=dropout)
self.r = nn.ReLU(inplace=True)
modules = nn.Sequential(*[self.model.features[0],
#self.bn64,
self.model.features[1:3],
#self.bn64,
self.model.features[3:5],
#self.dropout,
self.model.features[5],
#self.bn128,
self.model.features[6:8],
#self.bn128,
self.model.features[8:10],
#self.dropout,
self.model.features[10],
#self.bn256,
self.model.features[11:13],
#self.bn256,
self.model.features[13:15],
#self.bn256,
self.model.features[15:17],
#self.dropout,
self.model.features[17],
#self.bn512,
self.model.features[18:20],
#self.bn512,
self.model.features[20:22],
#self.bn512,
self.model.features[22:24],
#self.dropout,
self.model.features[24],
#self.bn512,
self.model.features[25:27],
#self.bn512,
self.model.features[27:29],
#self.bn512,
#self.dropout
nn.MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
])
self.model.features = modules
# change the pooling layer
self.model.avgpool = nn.AdaptiveAvgPool2d(output_size=(1, 1))
# set output to correct num classes
self.model.classifier = nn.Linear(in_features=512, out_features=n_classes, bias=True)
# use predefined bias
if bias is not None:
assert isinstance(bias, torch.Tensor), 'bias must be tensor'
self.model.classifier.bias = nn.Parameter(bias)
def forward(self, x):
x = self.model.features(x)
x = self.model.avgpool(x)
x = self.model.classifier(x)
return x
我知道它非常丑陋而且看起来很老套。我试图重写它，但由于某种原因，重写的版本也不起作用，我假设当前的问题也与此有关。我认为输入并没有像我认为的那样通过前向方法馈送。我的假设是调用 x = self.model(x) 不会 运行 'x' 通过我所做的所有编辑层，否则我会得到与上面两个版本的转发方法相同的行为。但是我的问题是，当我向前调用 self.model(x) 时发生了什么？是通过pytorch的原始vgg16输入运行吗？因为当我在控制台中打印 self.model 时，它显示了我对 self.model.features 以及 self.model.avgpool 和 self.model.classifier.
的架构所做的更改
编辑：
下面是错误的完整踪迹。一些额外的信息。 t 是我用来处理训练步骤的 class（因此循环训练和验证模式等）
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-7-0b3983ae9702> in <module>
77 print_cl_met = True
78 )
---> 79 model = t.run()
80 t.save_to_json()
81 print(np.max(np.array(t.f1_tracker)))
/home/stevea/treesat/TreeSat/TreeSat/trainers/basetrainer.py in run(self)
342 for phase in ['training', 'testing']:
343 self.phase = phase
--> 344 self.model_mode()
345
346 if self.phase == 'testing':
/home/stevea/treesat/TreeSat/TreeSat/trainers/basetrainer.py in model_mode(self)
154 if self.phase == 'training':
155 print('*********TRAINING PHASE*********')
--> 156 self.trainModel()
157 else:
158 print('*********VALIDATION PHASE*********')
/home/stevea/treesat/TreeSat/TreeSat/trainers/basetrainer.py in trainModel(self)
234 # loop through all batches to perform an epoch
235 for loaded in self.loaders[self.phase]:
--> 236 epoch_loss = self.train_step(loaded, epoch_loss)
237
238 mean_loss = np.mean(np.array(epoch_loss))
/home/stevea/treesat/TreeSat/TreeSat/trainers/basetrainer.py in train_step(self, loaded, epoch_loss)
262
263 # process batch through network
--> 264 self.out = self.model(self.img_batch.float())
265
266 # get loss value
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
887 result = self._slow_forward(*input, **kwargs)
888 else:
--> 889 result = self.forward(*input, **kwargs)
890 for hook in itertools.chain(
891 _global_forward_hooks.values(),
/home/stevea/treesat/TreeSat/TreeSat/models/vgg.py in forward(self, x)
143 x = self.model.features(x)
144 x = self.model.avgpool(x)
--> 145 x = self.model.classifier(x)
146 return x
147
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
887 result = self._slow_forward(*input, **kwargs)
888 else:
--> 889 result = self.forward(*input, **kwargs)
890 for hook in itertools.chain(
891 _global_forward_hooks.values(),
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/linear.py in forward(self, input)
92
93 def forward(self, input: Tensor) -> Tensor:
---> 94 return F.linear(input, self.weight, self.bias)
95
96 def extra_repr(self) -> str:
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in linear(input, weight, bias)
1751 if has_torch_function_variadic(input, weight):
1752 return handle_torch_function(linear, (input, weight), input, weight, bias=bias)
-> 1753 return torch._C._nn.linear(input, weight, bias)
1754
1755
RuntimeError: mat1 dim 1 must match mat2 dim 0
我还尝试在 forward 方法的每一步打印出 x 的形状：
def forward(self, x):
x = self.model.features(x)
print(x.shape)
x = self.model.avgpool(x)
print(x.shape)
x = self.model.classifier(x)
print(x.shape)
return x
它告诉我形状似乎很好，因为 classifier 应该接受 512 个特征：
torch.Size([64, 512, 4, 4])
torch.Size([64, 512, 1, 1])
我不能 运行 你的代码，但我认为问题是因为线性层需要 2d 数据输入（因为它实际上是一个矩阵乘法），而你提供 4d 输入（使用 dims 2 和3 码 1).
请尝试squeeze
def forward(self, x):
x = self.model.features(x)
x = self.model.avgpool(x)
x = torch.squeeze(x)
x = self.model.classifier(x)
return x
有关压缩部分中看起来不那么骇人听闻的代码，请参阅 torch einops。
我修改了 pytorch 中的 VGG16，以在特征提取器中插入 BN 和 dropout 等内容。当我更改 forward 方法的定义时，我偶然发现了一些奇怪的东西：
def forward(self, x):
x = self.model(x)
return x
至：
def forward(self, x):
x = self.model.features(x)
x = self.model.avgpool(x)
x = self.model.classifier(x)
return x
在第二种方法中，我现在收到矩阵大小不匹配的错误
(mat1 dim 1 must match mat2 dim 0)
下面是我一直在使用的 VGG 编辑版本的完整代码。
class Vgg(nn.Module):
def __init__(self, n_classes, bias= None, dropout = 0.3):
super().__init__()
self.model = models.vgg16()
#self.bn64 = nn.BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
#self.bn128 = nn.BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
#self.bn256 = nn.BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
#self.bn512 = nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
# change to allow 4 channels input
self.model.features[0] = nn.Conv2d(4, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3))
# remove/edit some of the first layers to make it more similar to Resnet
del self.model.features[2]
del self.model.features[2]
del self.model.features[-1]
self.model.features[2] = nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
# add dropout
for m in self.model.modules():
if isinstance(m, nn.Dropout):
m.p = dropout
else:
pass
self.dropout = nn.Dropout(p=dropout)
self.r = nn.ReLU(inplace=True)
modules = nn.Sequential(*[self.model.features[0],
#self.bn64,
self.model.features[1:3],
#self.bn64,
self.model.features[3:5],
#self.dropout,
self.model.features[5],
#self.bn128,
self.model.features[6:8],
#self.bn128,
self.model.features[8:10],
#self.dropout,
self.model.features[10],
#self.bn256,
self.model.features[11:13],
#self.bn256,
self.model.features[13:15],
#self.bn256,
self.model.features[15:17],
#self.dropout,
self.model.features[17],
#self.bn512,
self.model.features[18:20],
#self.bn512,
self.model.features[20:22],
#self.bn512,
self.model.features[22:24],
#self.dropout,
self.model.features[24],
#self.bn512,
self.model.features[25:27],
#self.bn512,
self.model.features[27:29],
#self.bn512,
#self.dropout
nn.MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
])
self.model.features = modules
# change the pooling layer
self.model.avgpool = nn.AdaptiveAvgPool2d(output_size=(1, 1))
# set output to correct num classes
self.model.classifier = nn.Linear(in_features=512, out_features=n_classes, bias=True)
# use predefined bias
if bias is not None:
assert isinstance(bias, torch.Tensor), 'bias must be tensor'
self.model.classifier.bias = nn.Parameter(bias)
def forward(self, x):
x = self.model.features(x)
x = self.model.avgpool(x)
x = self.model.classifier(x)
return x
我知道它非常丑陋而且看起来很老套。我试图重写它，但由于某种原因，重写的版本也不起作用，我假设当前的问题也与此有关。我认为输入并没有像我认为的那样通过前向方法馈送。我的假设是调用 x = self.model(x) 不会 运行 'x' 通过我所做的所有编辑层，否则我会得到与上面两个版本的转发方法相同的行为。但是我的问题是，当我向前调用 self.model(x) 时发生了什么？是通过pytorch的原始vgg16输入运行吗？因为当我在控制台中打印 self.model 时，它显示了我对 self.model.features 以及 self.model.avgpool 和 self.model.classifier.
编辑：
下面是错误的完整踪迹。一些额外的信息。 t 是我用来处理训练步骤的 class（因此循环训练和验证模式等）
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-7-0b3983ae9702> in <module>
77 print_cl_met = True
78 )
---> 79 model = t.run()
80 t.save_to_json()
81 print(np.max(np.array(t.f1_tracker)))
/home/stevea/treesat/TreeSat/TreeSat/trainers/basetrainer.py in run(self)
342 for phase in ['training', 'testing']:
343 self.phase = phase
--> 344 self.model_mode()
345
346 if self.phase == 'testing':
/home/stevea/treesat/TreeSat/TreeSat/trainers/basetrainer.py in model_mode(self)
154 if self.phase == 'training':
155 print('*********TRAINING PHASE*********')
--> 156 self.trainModel()
157 else:
158 print('*********VALIDATION PHASE*********')
/home/stevea/treesat/TreeSat/TreeSat/trainers/basetrainer.py in trainModel(self)
234 # loop through all batches to perform an epoch
235 for loaded in self.loaders[self.phase]:
--> 236 epoch_loss = self.train_step(loaded, epoch_loss)
237
238 mean_loss = np.mean(np.array(epoch_loss))
/home/stevea/treesat/TreeSat/TreeSat/trainers/basetrainer.py in train_step(self, loaded, epoch_loss)
262
263 # process batch through network
--> 264 self.out = self.model(self.img_batch.float())
265
266 # get loss value
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
887 result = self._slow_forward(*input, **kwargs)
888 else:
--> 889 result = self.forward(*input, **kwargs)
890 for hook in itertools.chain(
891 _global_forward_hooks.values(),
/home/stevea/treesat/TreeSat/TreeSat/models/vgg.py in forward(self, x)
143 x = self.model.features(x)
144 x = self.model.avgpool(x)
--> 145 x = self.model.classifier(x)
146 return x
147
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
887 result = self._slow_forward(*input, **kwargs)
888 else:
--> 889 result = self.forward(*input, **kwargs)
890 for hook in itertools.chain(
891 _global_forward_hooks.values(),
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/linear.py in forward(self, input)
92
93 def forward(self, input: Tensor) -> Tensor:
---> 94 return F.linear(input, self.weight, self.bias)
95
96 def extra_repr(self) -> str:
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in linear(input, weight, bias)
1751 if has_torch_function_variadic(input, weight):
1752 return handle_torch_function(linear, (input, weight), input, weight, bias=bias)
-> 1753 return torch._C._nn.linear(input, weight, bias)
1754
1755
RuntimeError: mat1 dim 1 must match mat2 dim 0
我还尝试在 forward 方法的每一步打印出 x 的形状：
def forward(self, x):
x = self.model.features(x)
print(x.shape)
x = self.model.avgpool(x)
print(x.shape)
x = self.model.classifier(x)
print(x.shape)
return x
它告诉我形状似乎很好，因为 classifier 应该接受 512 个特征：
torch.Size([64, 512, 4, 4])
torch.Size([64, 512, 1, 1])
我不能 运行 你的代码，但我认为问题是因为线性层需要 2d 数据输入（因为它实际上是一个矩阵乘法），而你提供 4d 输入（使用 dims 2 和3 码 1).
请尝试squeeze
def forward(self, x):
x = self.model.features(x)
x = self.model.avgpool(x)
x = torch.squeeze(x)
x = self.model.classifier(x)
return x
有关压缩部分中看起来不那么骇人听闻的代码，请参阅 torch einops。