I got a DCGAN and I want to modify it so the last TransConv2D layer output can be passed into a LSTM layer.
To test this, I got an array of images data with shape (2092, 64, 64, 3) which is also gonna be the input for the Neural Network . I’ve extracted the RGB arrays to pass each color array into the LSTM layer.
Here’s the TEST code, used outside the Neural Network:
Red = data[:,:,:,0]
outputR = []

for i in range(8, 64):
  outputR.append(Red[i-8:i, 0])

outputR = np.array(outputR)

Everything goes fine and I get outputR with shape (56,8,64)
However, when I try to apply this code to my Neural Network:
R = RGB[:,0,:,:] # [n_samples, RGB, height, width]
G = RGB[:,1,:,:]
B = RGB[:,2,:,:]

R = R.detach().cpu().numpy()
G = G.detach().cpu().numpy()
B = B.detach().cpu().numpy()
        
outputR = []
outputG = []
outputB = []

for i in range(8, 64):
  outputR.append(R[i-8:i,0])
  outputG.append(G[i-8:i,0])
  outputB.append(B[i-8:i,0])

outputR = np.array(outputR, dtype='float32')
outputG = np.array(outputG, dtype='float32')
outputB = np.array(outputB, dtype='float32')

I get the following error: ValueError: setting an array element with a sequence.
Why this difference? In both cases, I’m operating with sequences, arrays with a batch_size, yet, when I’m testing the operation outside a neural network, everything goes smoothly.
Here’s the complete code for the neural network Generator:
class Generator(nn.Module):
    def __init__(self):
        super(Generator, self).__init__()
        # input is Z(batch_size, 100, 1,1), going into a convolution
        self.transconv1 = nn.ConvTranspose2d( 100, 64 * 8, 4, 1, 0, bias=False)
        self.batchnorm1 = nn.BatchNorm2d(64 * 8)
        self.ReLU = nn.ReLU(True)
        # state size. (ngf*8) x 4 x 4
        self.transconv2 = nn.ConvTranspose2d(64 * 8, 64 * 4, 4, 2, 1, bias=False)
        self.batchnorm2 = nn.BatchNorm2d(64 * 4)
        # state size. (ngf*4) x 8 x 8
        self.transconv3 = nn.ConvTranspose2d( 64 * 4, 64 * 2, 4, 2, 1, bias=False)
        self.batchnorm3 = nn.BatchNorm2d(64 * 2)
        # state size. (ngf*2) x 16 x 16
        self.transconv4 = nn.ConvTranspose2d(64 * 2, 64, 4, 2, 1, bias=False)
        self.batchnorm4 = nn.BatchNorm2d(64)
        # state size. (ngf) x 32 x 32
        self.transconv5 = nn.ConvTranspose2d( 64, 3, 4, 2, 1, bias=False)
        self.lstm5 = nn.LSTM(input_size=64,hidden_size=64,num_layers=10, bias=False, batch_first=True)
        # state size. (nc) x 64 x 64

    def forward(self, input):
        x = self.transconv1(input)
        x = self.batchnorm1(x)
        x = self.ReLU(x)
        x = self.transconv2(x)
        x = self.batchnorm2(x)
        x = self.ReLU(x)
        x = self.transconv3(x)
        x = self.batchnorm3(x)
        x = self.transconv4(x)
        x = self.batchnorm4(x)
        x = self.transconv5(x)
        R, G, B = self._preprocess(x)
        outR = self.lstm5(R)
        outG = self.lstm5(G)
        outB = self.lstm5(B)
        output = np.stack((outR, outG, outB), axis=1)

        return output

    def _preprocess(self, RGB):
        R = RGB[:,0,:,:]
        G = RGB[:,1,:,:]
        B = RGB[:,2,:,:]

        R = R.detach().cpu().numpy()
        G = G.detach().cpu().numpy()
        B = B.detach().cpu().numpy()
        print(R.shape)
        
        outputR = []
        outputG = []
        outputB = []

        for i in range(8, 64):
            outputR.append(R[i-8:i,0])
            outputG.append(G[i-8:i,0])
            outputB.append(B[i-8:i,0])

        outputR = np.array(outputR, dtype='float32')
        outputG = np.array(outputG, dtype='float32')
        outputB = np.array(outputB, dtype='float32')

        #(n_samples, timesteps, features)
        return torch.from_numpy(outputR), torch.from_numpy(outputG), torch.from_numpy(outputB)