By the way, here are all relevant parts of my model:
ysize = 45
zsize = 100
isize = 128

batchsize = 16

gen_up_blocks = 4
gen_inp_planes = 32
gen_resblocks = 3
gen_filters = 256
gen_self_attn = [0, 1, 0, 0, 0]

class Generator(nn.Module):
    def __init__(self, isize, ysize, zsize, gen_up_blocks, gen_inp_planes,
                 gen_resblocks, gen_filters, gen_self_attn):
        super(Generator, self).__init__()
        sidedim = isize // (gen_up_blocks ** 2)
        self.sidedim = sidedim
        self.latent = nn.Linear(zsize + ysize, (sidedim ** 2) * gen_inp_planes)
        main = nn.Sequential()

        main.add_module('pad_init', nn.ReflectionPad2d(1))
        main.add_module('conv_init', nn.utils.spectral_norm(nn.Conv2d(gen_inp_planes, gen_filters, 3))())

        if gen_self_attn[0]:
            main.add_module('sa_{}'.format(0), Self_Attn(gen_filters))


        for i in range(gen_up_blocks):
            mod = GenUpBlock(gen_filters, gen_filters // 2, gen_resblocks)

            main.add_module('up_{}'.format(i), mod)

            gen_filters //= 2

            if gen_self_attn[i + 1]:
                main.add_module('sa_{}'.format(i + 1), Self_Attn(gen_filters))

        main.add_module('padding', nn.ReflectionPad2d(3))
        main.add_module('conv_final', nn.utils.spectral_norm(nn.Conv2d(gen_filters, 3, 7))())
        main.add_module('sigmoid', nn.Sigmoid())
        self.gen_inp_planes = gen_inp_planes
        self.main = main

    def forward(self, z, y):
        z = self.latent(torch.cat((z, y), 1))
        z = z.view(-1, self.gen_inp_planes, self.sidedim, self.sidedim)
        z = self.main(z)

        return z

class ResBlock(nn.Module):
    def __init__(self, channels, bn):
        super(ResBlock, self).__init__()

        self.bn = bn

        self.pad1 = nn.ReflectionPad2d(1)
        self.conv1 = nn.utils.spectral_norm(nn.Conv2d(channels, channels, 3))()
        if bn:
            self.bn1 = nn.BatchNorm2d(channels)
        self.pad2 = nn.ReflectionPad2d(1)
        self.conv2 = nn.utils.spectral_norm(nn.Conv2d(channels, channels, 3))()
        if bn:
            self.bn2 = nn.BatchNorm2d(channels)

    def forward(self, x):
        residual = x
        out = self.pad1(x)
        out = self.conv1(out)
        if self.bn:
            out = self.bn1(out)
        out = nn.functional.leaky_relu(out, negative_slope=0.2)
        out = self.pad2(out)
        out = self.conv2(out)
        if self.bn:
            out = self.bn2(out)
        out += residual
        out = nn.functional.leaky_relu(out, negative_slope=0.2)
        return out

class GenUpBlock(nn.Module):
    def __init__(self, in_channels, out_channels, resblocks):
        super(GenUpBlock, self).__init__()

        self.res = nn.Sequential()
        for i in range(resblocks):
            self.res.add_module('res_{}'.format(i), ResBlock(in_channels, True))
        self.pad = nn.ReflectionPad2d(1)
        self.conv = nn.utils.spectral_norm(nn.Conv2d(in_channels, out_channels * 4, 3))()
        self.shuf = nn.PixelShuffle(2)
        self.bn = nn.BatchNorm2d(out_channels)
        self.relu = nn.LeakyReLU(0.2)

    def forward(self, x):
        skip = x
        x = self.res(x)
        x += skip
        x = self.pad(x)
        x = self.conv(x)
        x = self.shuf(x)
        x = self.bn(x)
        x = self.relu(x)
        return x

# (Modified) From https://github.com/heykeetae/Self-Attention-GAN/blob/master/sagan_models.py
class Self_Attn(nn.Module):
    """ Self attention Layer"""

    def __init__(self, in_dim):
        super(Self_Attn, self).__init__()

        self.kernel_size = 1

        self.query_conv = nn.utils.spectral_norm(nn.Conv2d(
            in_channels=in_dim, out_channels=in_dim // 8, kernel_size=self.kernel_size))
        self.key_conv = nn.utils.spectral_norm(nn.Conv2d(
            in_channels=in_dim, out_channels=in_dim // 8, kernel_size=self.kernel_size))
        self.value_conv = nn.utils.spectral_norm(nn.Conv2d(
            in_channels=in_dim, out_channels=in_dim, kernel_size=self.kernel_size))
        self.gamma = nn.Parameter(torch.zeros(1))

        self.softmax = nn.Softmax(dim=-1)

    def forward(self, x):
        """
            inputs :
                x : input feature maps( B X C X W X H)
            returns :
                out : self attention value + input feature
                attention: B X N X N (N is Width*Height)
        """
        m_batchsize, C, width, height = x.size()

        px = x

        proj_query = self.query_conv(px).view(
            m_batchsize, -1, width * height).permute(0, 2, 1)  # B X CX(N)
        proj_key = self.key_conv(px).view(
            m_batchsize, -1, width * height)  # B X C x (*W*H)
        energy = torch.bmm(proj_query, proj_key)  # transpose check
        attention = self.softmax(energy)  # BX (N) X (N)
        proj_value = self.value_conv(px).view(
            m_batchsize, -1, width * height)  # B X C X N

        out = torch.bmm(proj_value, attention.permute(0, 2, 1))
        out = out.view(m_batchsize, C, width, height)

        out = self.gamma * out + x
        return out