Hi,
I have a problem with batchnorm(1d) there are some discussions on that but did’t helped
I train a network (with batchnorm1d) with batch_size 512. during eval and train it works fine with same number of batches but by changing batchsize from 512 to another value accuracy decreases
network:
class Model(nn.Module):
	def __init__(self, outdim, momentum, track=False):
		super(Model, self).__init__()
		self.inp = nn.Sequential(
              nn.BatchNorm1d(256, track_running_stats=track, momentum=momentum),
        )
		self.rnn1 = nn.GRU(7, 32, batch_first=True, bidirectional=True)
		self.attn1 = nn.MultiheadAttention(32*2, 8, dropout=.2)
		self.model = nn.Sequential(
			nn.BatchNorm1d(256*32*2, track_running_stats=track, momentum=momentum),
			nn.Linear(256*32*2, 64), nn.ReLU(),
			nn.BatchNorm1d(64, track_running_stats=track, momentum=momentum),
			nn.Linear(64, 128), nn.ReLU(),
			nn.BatchNorm1d(128, track_running_stats=track, momentum=momentum),
			nn.Linear(128, 128), nn.ReLU(),
		)
		self.out = nn.Sequential(
			nn.BatchNorm1d(128, track_running_stats=track, momentum=momentum),
			nn.Linear(128, outdim)
		)
		self.opt = AdamW(self.parameters(), .0001)
		self.loss = nn.CrossEntropyLoss()
		self.to(device)

	def forward(self, x):
		x = T.as_tensor(x).to(device).float()# x: (-1, 256, 7)
		x = self.inp(x)
		rnn1 = self.rnn1(x)[0]
		attn1 = self.attn1(*[rnn1.transpose(0,1)]*3)[0].transpose(0,1)
		model = self.model(attn1.flatten(1))
		return self.out(model)


results:
after doing following:
model2 = Model(2, 0)
model2.load_state_dict(model.state_dict())
with T.no_grad():
    model2.forward(x_train[:32])
    model2.forward(x_train[32:32*2])
    model2.forward(x_train[32*2:32*3])
    model2.forward(x_train[32*3:32*4])
model2.eval()
with T.no_grad():
    accs = []
    for j,(x,y) in enumerate(data.batch(x=x_train, y=y_train, batch_size=32)):
        output = model2(x).cpu().detach().numpy().argmax(-1).reshape(-1)
        accs.append((output == y.astype(int)).sum() / len(y))
        print(str(j/(len(y_train) / x.shape[0]))[:7])
        display.clear_output(1)
print(str(np.mean(accs))[:7])

Accuracy: 85.5%
and if I change batch_size from 512 to 32 results in:
Acuraccy: 47.5%
btw I also tried changing momentum in model2 but no effects