In this paper: https://arxiv.org/pdf/2004.02967.pdf 6 the authors use evolutionary algorithms to find better activation-normalization layers.
They provide this tensorflow pseudocode in the appendix (page 11)
def evonorm_b0(x, gamma, beta, nonlinearity, training):
  if nonlinearity:
    v = trainable_variable_ones(shape=gamma.shape)
    _, batch_std = batch_mean_and_std(x, training)
    den = tf.maximum(batch_std, v * x + instance_std(x))
    return x / den * gamma + beta
  else:
    return x * gamma + beta

# Helper functions
def instance_std(x, eps=1e−5):
  _, var = tf.nn.moments(x, axes=[1, 2], keepdims=True)
  return tf.sqrt(var + eps)

def group_std(x, groups=32, eps=1e−5):
  N, H, W, C = x.shape
  x = tf.reshape(x, [N, H, W, groups, C // groups])
  _, var = tf.nn.moments(x, [1, 2, 4], keepdims=True)
  return tf.reshape(tf.sqrt(var + eps), [N, H, W, C])

def trainable_variable_ones(shape, name="v"):
  return tf.get_variable(
    name, shape=shape, initializer=tf.ones_initializer()
  )

What would be the pytorch equivalent for the evonorm_b0 (and helper functions) ? I want to make sure I don’t mess my implementation.